From 4062dc70fb196e6b85c4ada990d541c01aab6875 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 15:03:55 +0200 Subject: [PATCH 001/156] Initial export/import API for devices, device profiles, assets, rule chains, dashboards and customers --- .../main/data/upgrade/3.4/schema_update.sql | 28 +++ .../EntitiesExportImportController.java | 99 +++++++++ .../DefaultEntitiesExportImportService.java | 86 ++++++++ .../expimp/EntitiesExportImportService.java | 29 +++ .../expimp/exp/EntityExportService.java | 32 +++ .../expimp/exp/impl/AssetExportService.java | 50 +++++ .../exp/impl/CustomerExportService.java | 50 +++++ .../exp/impl/DashboardExportService.java | 50 +++++ .../expimp/exp/impl/DeviceExportService.java | 53 +++++ .../exp/impl/DeviceProfileExportService.java | 50 +++++ .../exp/impl/RuleChainExportService.java | 51 +++++ .../expimp/imp/EntityImportService.java | 30 +++ .../imp/impl/AbstractEntityImportService.java | 73 +++++++ .../expimp/imp/impl/AssetImportService.java | 62 ++++++ .../imp/impl/CustomerImportService.java | 60 ++++++ .../imp/impl/DashboardImportService.java | 62 ++++++ .../expimp/imp/impl/DeviceImportService.java | 81 ++++++++ .../imp/impl/DeviceProfileImportService.java | 65 ++++++ .../imp/impl/RuleChainImportService.java | 80 ++++++++ .../server/dao/rule/RuleChainService.java | 3 +- .../server/common/data/Customer.java | 6 + .../server/common/data/Dashboard.java | 10 +- .../server/common/data/Device.java | 7 + .../server/common/data/DeviceProfile.java | 3 + .../server/common/data/asset/Asset.java | 7 + .../data/export/EntitiesExportRequest.java | 29 +++ .../data/export/EntitiesExportResponse.java | 29 +++ .../common/data/export/EntityExportData.java | 48 +++++ .../data/export/impl/AssetExportData.java | 33 +++ .../data/export/impl/CustomerExportData.java | 33 +++ .../data/export/impl/DashboardExportData.java | 33 +++ .../data/export/impl/DeviceExportData.java | 35 ++++ .../export/impl/DeviceProfileExportData.java | 33 +++ .../data/export/impl/RuleChainExportData.java | 35 ++++ .../server/common/data/rule/RuleChain.java | 3 + .../server/dao/ExportableEntityDao.java | 36 ++++ .../dao/ExportableEntityRepository.java | 26 +++ .../server/dao/asset/AssetDao.java | 4 +- .../server/dao/customer/CustomerDao.java | 5 +- .../server/dao/dashboard/DashboardDao.java | 3 +- .../server/dao/device/DeviceDao.java | 3 +- .../server/dao/device/DeviceProfileDao.java | 3 +- .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AbstractAssetEntity.java | 11 + .../dao/model/sql/AbstractDeviceEntity.java | 10 + .../server/dao/model/sql/CustomerEntity.java | 9 + .../server/dao/model/sql/DashboardEntity.java | 9 + .../dao/model/sql/DeviceProfileEntity.java | 9 + .../server/dao/model/sql/RuleChainEntity.java | 9 + .../server/dao/rule/BaseRuleChainService.java | 5 + .../server/dao/rule/RuleChainDao.java | 7 +- .../server/dao/sql/asset/AssetRepository.java | 3 +- .../server/dao/sql/asset/JpaAssetDao.java | 16 ++ .../dao/sql/customer/CustomerRepository.java | 3 +- .../dao/sql/customer/JpaCustomerDao.java | 17 ++ .../sql/dashboard/DashboardRepository.java | 4 +- .../dao/sql/dashboard/JpaDashboardDao.java | 18 ++ .../sql/device/DeviceProfileRepository.java | 4 +- .../dao/sql/device/DeviceRepository.java | 3 +- .../server/dao/sql/device/JpaDeviceDao.java | 16 ++ .../dao/sql/device/JpaDeviceProfileDao.java | 17 ++ .../server/dao/sql/rule/JpaRuleChainDao.java | 16 ++ .../dao/sql/rule/RuleChainRepository.java | 3 +- .../main/resources/sql/schema-entities.sql | 188 +++++++++--------- 64 files changed, 1788 insertions(+), 109 deletions(-) create mode 100644 application/src/main/data/upgrade/3.4/schema_update.sql create mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java diff --git a/application/src/main/data/upgrade/3.4/schema_update.sql b/application/src/main/data/upgrade/3.4/schema_update.sql new file mode 100644 index 0000000000..ad307a6576 --- /dev/null +++ b/application/src/main/data/upgrade/3.4/schema_update.sql @@ -0,0 +1,28 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +ALTER TABLE device + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE device_profile + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE asset + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_chain + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE dashboard + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE customer + ADD COLUMN IF NOT EXISTS external_id UUID; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java new file mode 100644 index 0000000000..a11a9189f4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -0,0 +1,99 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.export.EntitiesExportRequest; +import org.thingsboard.server.common.data.export.EntitiesExportResponse; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.EntitiesExportImportService; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@RestController +@RequestMapping("/api/entities") +@TbCoreComponent +@RequiredArgsConstructor +public class EntitiesExportImportController extends BaseController { + + private final EntitiesExportImportService exportImportService; + + + @PostMapping("/export/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityExportData exportEntity(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid) throws ThingsboardException { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + try { + return exportImportService.exportEntity(getTenantId(), entityId); + } catch (Exception e) { + throw handleException(e); + } + } + + @PostMapping("/export/batch") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { + TenantId tenantId = getTenantId(); + + EntitiesExportResponse exportResponse = new EntitiesExportResponse(); + + Map>>> result = new HashMap<>(); + exportRequest.getEntities().forEach((entityType, entityIds) -> { + List>> exportDataForEntityType = new LinkedList<>(); + entityIds.forEach(entityId -> { + EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); + exportDataForEntityType.add(exportData); + }); + result.put(entityType, exportDataForEntityType); + }); + + exportResponse.setExportData(result); + return exportResponse; + } + // TODO: api to export and import whole customer, whole tenant + + + @PostMapping("/import") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public , I extends EntityId, D extends EntityExportData> E importEntity(@RequestBody D exportData) throws ThingsboardException { + try { + return exportImportService.importEntity(getTenantId(), exportData); + } catch (Exception e) { + throw handleException(e); + } + } + +// public void importEntities(@RequestBody ) + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java new file mode 100644 index 0000000000..9b384b34dd --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.imp.EntityImportService; + +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; + +// FIXME: review packages and classes naming +@Service +@TbCoreComponent +public class DefaultEntitiesExportImportService implements EntitiesExportImportService { + + private final Map> exportServices = new EnumMap<>(EntityType.class); + private final Map> importServices = new EnumMap<>(EntityType.class); + + + // TODO: export and import of the whole tenant + // TODO: export and import of the whole customer ? + // TODO: relations export and import + @Override + public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { + EntityType entityType = entityId.getEntityType(); + EntityExportService exportService = getExportService(entityType); + + return exportService.getExportData(tenantId, entityId); + } + + // FIXME: somehow validate export data + // FIXME: validate permissions for create or update + // FIXME: send entity lifecycle event + @Override + public , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData) { + EntityType entityType = exportData.getEntityType(); + EntityImportService importService = getImportService(entityType); + + return importService.importEntity(tenantId, exportData); + } + + + @SuppressWarnings("unchecked") + private > EntityExportService getExportService(EntityType entityType) { + return (EntityExportService) exportServices.get(entityType); + } + + @SuppressWarnings("unchecked") + private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + return (EntityImportService) importServices.get(entityType); + } + + @Autowired + private void setExportImportServices(Collection> exportServices, + Collection> importServices) { + exportServices.forEach(entityExportService -> { + this.exportServices.put(entityExportService.getEntityType(), entityExportService); + }); + importServices.forEach(entityImportService -> { + this.importServices.put(entityImportService.getEntityType(), entityImportService); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java new file mode 100644 index 0000000000..5d25cb9320 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp; + +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EntitiesExportImportService { + + , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); + + , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java new file mode 100644 index 0000000000..9afa01ff8e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EntityExportService> { + + // FIXME: export relations + // FIXME: get rid of boilerplate + EntityExportData getExportData(TenantId tenantId, I entityId); + + EntityType getEntityType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java new file mode 100644 index 0000000000..bb4b68bb29 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.AssetExportData; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetExportService implements EntityExportService { + + private final AssetService assetService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, AssetId assetId) { + AssetExportData exportData = new AssetExportData(); + exportData.setAsset(assetService.findAssetById(tenantId, assetId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java new file mode 100644 index 0000000000..7430f9ddd3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.CustomerExportData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CustomerExportService implements EntityExportService { + + private final CustomerService customerService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, CustomerId customerId) { + CustomerExportData exportData = new CustomerExportData(); + exportData.setCustomer(customerService.findCustomerById(tenantId, customerId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java new file mode 100644 index 0000000000..87959429d9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.DashboardExportData; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DashboardExportService implements EntityExportService { + + private final DashboardService dashboardService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, DashboardId dashboardId) { + DashboardExportData exportData = new DashboardExportData(); + exportData.setDashboard(dashboardService.findDashboardById(tenantId, dashboardId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java new file mode 100644 index 0000000000..860aaa9b1d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.DeviceExportData; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceExportService implements EntityExportService { + + private final DeviceService deviceService; + private final DeviceCredentialsService deviceCredentialsService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, DeviceId deviceId) { + DeviceExportData exportData = new DeviceExportData(); + exportData.setDevice(deviceService.findDeviceById(tenantId, deviceId)); + exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java new file mode 100644 index 0000000000..425dbfd5be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceProfileExportService implements EntityExportService { + + private final DeviceProfileService deviceProfileService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, DeviceProfileId deviceProfileId) { + DeviceProfileExportData exportData = new DeviceProfileExportData(); + exportData.setDeviceProfile(deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java new file mode 100644 index 0000000000..7a85281d63 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.exp.EntityExportService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainExportService implements EntityExportService { + + private final RuleChainService ruleChainService; + + + @Override + public EntityExportData getExportData(TenantId tenantId, RuleChainId ruleChainId) { + RuleChainExportData exportData = new RuleChainExportData(); + exportData.setRuleChain(ruleChainService.findRuleChainById(tenantId, ruleChainId)); + exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId)); + return exportData; + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java new file mode 100644 index 0000000000..195ace0b66 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface EntityImportService, D extends EntityExportData> { + + E importEntity(TenantId tenantId, D exportData); + + EntityType getEntityType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java new file mode 100644 index 0000000000..3f33ba04f0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -0,0 +1,73 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.service.expimp.imp.EntityImportService; + +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; + +public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { + + private final Map> daos = new EnumMap<>(EntityType.class); + + + public final E findByExternalId(TenantId tenantId, I externalId) { + return findByExternalOrInternalId(tenantId, externalId); + } + + + protected final ID getInternalId(TenantId tenantId, ID externalId) { + if (externalId == null) { + return null; + } + HasId entity = findByExternalOrInternalId(tenantId, externalId); + if (entity == null) { + throw new IllegalStateException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); + } + return entity.getId(); + } + + private , ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { + ExportableEntityDao dao = getDao(externalOrInternalId.getEntityType()); + return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalOrInternalId.getId())) + .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalOrInternalId.getId())); + } + + + @SuppressWarnings("unchecked") + private ExportableEntityDao getDao(EntityType entityType) { + return (ExportableEntityDao) daos.get(entityType); + } + + @Autowired + private void setDaos(Collection> daos) { + daos.forEach(dao -> this.daos.put(dao.getEntityType(), dao)); + if (!this.daos.containsKey(getEntityType())) { + throw new IllegalStateException(getClass().getSimpleName() + " requires ExportableEntityDao for entity type " + getEntityType()); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java new file mode 100644 index 0000000000..5e563762d6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.export.impl.AssetExportData; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetImportService extends AbstractEntityImportService { + + private final AssetService assetService; + + + @Override + public Asset importEntity(TenantId tenantId, AssetExportData exportData) { + Asset asset = exportData.getAsset(); + Asset existingAsset = findByExternalId(tenantId, asset.getId()); // TODO: extract boiler plate to abstract class ... + + asset.setExternalId(asset.getId()); + asset.setTenantId(tenantId); + + if (existingAsset == null) { + asset.setId(null); + } else { + asset.setId(existingAsset.getId()); + } + + asset.setCustomerId(getInternalId(tenantId, asset.getCustomerId())); + + Asset savedAsset = assetService.saveAsset(asset); + + return savedAsset; + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java new file mode 100644 index 0000000000..9510cc0700 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.CustomerExportData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CustomerImportService extends AbstractEntityImportService { + + private final CustomerService customerService; + + + @Override + public Customer importEntity(TenantId tenantId, CustomerExportData exportData) { + Customer customer = exportData.getCustomer(); + Customer existingCustomer = findByExternalId(tenantId, customer.getId()); + + customer.setExternalId(customer.getId()); + customer.setTenantId(tenantId); + + if (existingCustomer == null) { + customer.setId(null); + } else { + customer.setId(existingCustomer.getId()); + } + + Customer savedCustomer = customerService.saveCustomer(customer); + + return savedCustomer; + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java new file mode 100644 index 0000000000..eab90bb294 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.DashboardExportData; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DashboardImportService extends AbstractEntityImportService { + + private final DashboardService dashboardService; + + + @Override + public Dashboard importEntity(TenantId tenantId, DashboardExportData exportData) { + Dashboard dashboard = exportData.getDashboard(); + Dashboard existingDashboard = findByExternalId(tenantId, dashboard.getId()); + + dashboard.setExternalId(dashboard.getId()); + dashboard.setTenantId(tenantId); + + if (existingDashboard == null) { + dashboard.setId(null); + dashboard.setAssignedCustomers(null); // FIXME: need to assign dashboard to customers ? + } else { + dashboard.setId(existingDashboard.getId()); + dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); // we left them untouched (FIXME) + } + + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); + + return savedDashboard; + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java new file mode 100644 index 0000000000..052fa3e1b3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.DeviceExportData; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.action.EntityActionService; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceImportService extends AbstractEntityImportService { + + private final DeviceService deviceService; + + + @Transactional + @Override + public Device importEntity(TenantId tenantId, DeviceExportData exportData) { + Device device = exportData.getDevice(); + Device existingDevice = findByExternalId(tenantId, device.getId()); // FIXME: !!! + // what if exporting and importing back already exported entity ? (save version and then load it back) + /* + * export entity -> id from env1 -> import this entity -> ... + * + * maybe find not only by external id but by internal too ? but then what if we will try + * */ + + device.setExternalId(device.getId()); + device.setTenantId(tenantId); + + if (existingDevice == null) { + device.setId(null); + device.setCustomerId(null); // FIXME: find and set customer + } else { + device.setId(existingDevice.getId()); + device.setCustomerId(existingDevice.getCustomerId()); + } + + // TODO or maybe set as additional config whether to update related entities when device already exists ? + // TODO: or also whether to ignore not found internal ids + + // FIXME: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants + + device.setDeviceProfileId(getInternalId(tenantId, device.getDeviceProfileId())); + device.setFirmwareId(getInternalId(tenantId, device.getFirmwareId())); + device.setSoftwareId(getInternalId(tenantId, device.getSoftwareId())); + + Device savedDevice = deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); + + return savedDevice; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java new file mode 100644 index 0000000000..615f62d389 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceProfileImportService extends AbstractEntityImportService { + + private final DeviceProfileService deviceProfileService; + + + @Override + public DeviceProfile importEntity(TenantId tenantId, DeviceProfileExportData exportData) { + DeviceProfile deviceProfile = exportData.getDeviceProfile(); + DeviceProfile existingDeviceProfile = findByExternalId(tenantId, deviceProfile.getId()); + + deviceProfile.setExternalId(deviceProfile.getId()); + deviceProfile.setTenantId(tenantId); + + if (existingDeviceProfile == null) { + deviceProfile.setId(null); + } else { + deviceProfile.setId(existingDeviceProfile.getId()); + } + + deviceProfile.setDefaultRuleChainId(getInternalId(tenantId, deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(getInternalId(tenantId, deviceProfile.getDefaultDashboardId())); + deviceProfile.setFirmwareId(getInternalId(tenantId, deviceProfile.getFirmwareId())); + deviceProfile.setSoftwareId(getInternalId(tenantId, deviceProfile.getSoftwareId())); + + DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); + + return savedDeviceProfile; + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java new file mode 100644 index 0000000000..eb72f58a7f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainImportService extends AbstractEntityImportService { + + private final RuleChainService ruleChainService; + + + @Transactional + @Override + public RuleChain importEntity(TenantId tenantId, RuleChainExportData exportData) { + RuleChain ruleChain = exportData.getRuleChain(); + RuleChain existingRuleChain = findByExternalId(tenantId, ruleChain.getId()); + + ruleChain.setExternalId(ruleChain.getId()); + ruleChain.setTenantId(tenantId); + ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting + + if (existingRuleChain == null) { + ruleChain.setId(null); + } else { + ruleChain.setId(existingRuleChain.getId()); + } + + RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain); + + if (ruleChain.getId() != null) { + ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); + } + RuleChainMetaData metaData = exportData.getMetaData(); + metaData.setRuleChainId(savedRuleChain.getId()); + metaData.getNodes().forEach(ruleNode -> { + ruleNode.setId(null); + ruleNode.setRuleChainId(null); + }); + metaData.getRuleChainConnections().forEach(ruleChainConnectionInfo -> { +// ruleChainConnectionInfo.setTargetRuleChainId(); + // TODO: check if this thing is needed for "Other Rule Chain Node" + // TODO: and check import of tenant rule chains + }); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return savedRuleChain; + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 5089a6286e..0c42b8aec8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainData; import org.thingsboard.server.common.data.rule.RuleChainImportResult; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.data.rule.RuleNode; @@ -93,4 +92,6 @@ public interface RuleChainService { List findRuleNodesByTenantIdAndType(TenantId tenantId, String name, String toString); RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode); + + void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index ab21d70795..065e0c5db5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; @@ -36,6 +38,9 @@ public class Customer extends ContactBased implements HasTenantId { @ApiModelProperty(position = 5, required = true, value = "JSON object with Tenant Id") private TenantId tenantId; + @Getter @Setter + private CustomerId externalId; // FIXME: add to hashcode, equals, etc + public Customer() { super(); } @@ -48,6 +53,7 @@ public class Customer extends ContactBased implements HasTenantId { super(customer); this.tenantId = customer.getTenantId(); this.title = customer.getTitle(); + this.externalId = customer.getExternalId(); } public TenantId getTenantId() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 450c88cbb5..4b69534bc0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,14 +17,19 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; public class Dashboard extends DashboardInfo { private static final long serialVersionUID = 872682138346187503L; - + private transient JsonNode configuration; - + + @Getter @Setter + private DashboardId externalId; + public Dashboard() { super(); } @@ -40,6 +45,7 @@ public class Dashboard extends DashboardInfo { public Dashboard(Dashboard dashboard) { super(dashboard); this.configuration = dashboard.getConfiguration(); + this.externalId = dashboard.getExternalId(); } @ApiModelProperty(position = 9, value = "JSON object with main configuration of the dashboard: layouts, widgets, aliases, etc. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 4d7bf5e822..519cffc39a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.id.CustomerId; @@ -61,6 +63,9 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen private OtaPackageId firmwareId; private OtaPackageId softwareId; + @Getter @Setter + private DeviceId externalId; + public Device() { super(); } @@ -80,6 +85,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setDeviceData(device.getDeviceData()); this.firmwareId = device.getFirmwareId(); this.softwareId = device.getSoftwareId(); + this.externalId = device.getExternalId(); } public Device updateDevice(Device device) { @@ -93,6 +99,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setFirmwareId(device.getFirmwareId()); this.setSoftwareId(device.getSoftwareId()); Optional.ofNullable(device.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.setExternalId(device.getExternalId()); return this; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 1dd6a2ad19..6161cd6a5c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -90,6 +90,8 @@ public class DeviceProfile extends SearchTextBased implements H @ApiModelProperty(position = 10, value = "Reference to the software OTA package. If present, the specified package will be used as default device software. ") private OtaPackageId softwareId; + private DeviceProfileId externalId; + public DeviceProfile() { super(); } @@ -112,6 +114,7 @@ public class DeviceProfile extends SearchTextBased implements H this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); this.firmwareId = deviceProfile.getFirmwareId(); this.softwareId = deviceProfile.getSoftwareId(); + this.externalId = deviceProfile.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the device profile Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index db68df4901..e0924c63ee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -49,6 +51,9 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements @Length(fieldName = "label") private String label; + @Getter @Setter + private AssetId externalId; + public Asset() { super(); } @@ -64,6 +69,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.name = asset.getName(); this.type = asset.getType(); this.label = asset.getLabel(); + this.externalId = asset.getExternalId(); } public void update(Asset asset) { @@ -73,6 +79,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.type = asset.getType(); this.label = asset.getLabel(); Optional.ofNullable(asset.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.externalId = asset.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the asset Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java new file mode 100644 index 0000000000..d3502695c0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; +import java.util.Map; + +@Data +public class EntitiesExportRequest { + // todo: should be processed in a specific order, e.g. device profiles, then devices, then relations, etc. + private Map> entities; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java new file mode 100644 index 0000000000..cab75313d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; + +import java.util.List; +import java.util.Map; + +@Data +public class EntitiesExportResponse { + private Map>>> exportData; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java new file mode 100644 index 0000000000..36d3cb4b0c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.impl.AssetExportData; +import org.thingsboard.server.common.data.export.impl.CustomerExportData; +import org.thingsboard.server.common.data.export.impl.DashboardExportData; +import org.thingsboard.server.common.data.export.impl.DeviceExportData; +import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; +import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) +@JsonSubTypes({ + @Type(name = "DEVICE", value = DeviceExportData.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), + @Type(name = "ASSET", value = AssetExportData.class), + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), + @Type(name = "DASHBOARD", value = DashboardExportData.class), + @Type(name = "CUSTOMER", value = CustomerExportData.class) +}) +public interface EntityExportData> { + + @JsonIgnore + EntityType getEntityType(); // fixme: maybe remove if not needed, as well as generic + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java new file mode 100644 index 0000000000..c0f417bd76 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class AssetExportData implements EntityExportData { + + private Asset asset; + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java new file mode 100644 index 0000000000..31e53db223 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class CustomerExportData implements EntityExportData { + + private Customer customer; + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java new file mode 100644 index 0000000000..7860c6c553 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class DashboardExportData implements EntityExportData { + + private Dashboard dashboard; + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java new file mode 100644 index 0000000000..05f3299d02 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +@Data +public class DeviceExportData implements EntityExportData { + + private Device device; + private DeviceCredentials credentials; + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java new file mode 100644 index 0000000000..c030d0b86e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; + +@Data +public class DeviceProfileExportData implements EntityExportData { + + private DeviceProfile deviceProfile; + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java new file mode 100644 index 0000000000..11fbace911 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; + +@Data +public class RuleChainExportData implements EntityExportData { + + private RuleChain ruleChain; + private RuleChainMetaData metaData; + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 17d7ffef0b..7ff2b0dc2b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -59,6 +59,8 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im @JsonIgnore private byte[] configurationBytes; + private RuleChainId externalId; + public RuleChain() { super(); } @@ -75,6 +77,7 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im this.firstRuleNodeId = ruleChain.getFirstRuleNodeId(); this.root = ruleChain.isRoot(); this.setConfiguration(ruleChain.getConfiguration()); + this.setExternalId(ruleChain.getExternalId()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java new file mode 100644 index 0000000000..c7741ec013 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.EntityType; + +import java.util.UUID; + +public interface ExportableEntityDao { + + T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + + T findByTenantIdAndId(UUID tenantId, UUID id); + // fixme: get rid of boilerplate ? + + EntityType getEntityType(); + + /* + * default > ExportableEntityRepository getExportableEntityRepository() { + * ((ExportableEntityRepository) getJpaRepository).find... + * */ + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java new file mode 100644 index 0000000000..b6a303a941 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import java.util.UUID; + +public interface ExportableEntityRepository { + + D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + + D findByTenantIdAndId(UUID tenantId, UUID id); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index e7533cc736..495886bcc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -22,8 +22,8 @@ import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -34,7 +34,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index d79537e564..89672d2a73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Optional; @@ -28,7 +29,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object @@ -37,7 +38,7 @@ public interface CustomerDao extends Dao, TenantEntityDao { * @return saved customer object */ Customer save(TenantId tenantId, Customer customer); - + /** * Find customers by tenant id and page link. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index dfe9152484..90353f49b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -18,12 +18,13 @@ package org.thingsboard.server.dao.dashboard; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 85f66e4b67..90f2b218a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -35,7 +36,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java index bf61cbdce5..b79cbb05ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java @@ -21,10 +21,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.UUID; -public interface DeviceProfileDao extends Dao { +public interface DeviceProfileDao extends Dao, ExportableEntityDao { DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, UUID deviceProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 8f692d5a12..0b74e79f2b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -559,6 +559,8 @@ public class ModelConstants { public static final String EDGE_EVENT_BY_ID_VIEW_NAME = "edge_event_by_id"; + public static final String EXTERNAL_ID_PROPERTY = "external_id"; + /** * Cassandra attributes and timeseries constants. */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index 4405768f7c..0936ca2855 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -38,6 +38,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ASSET_LABEL_PROPER import static org.thingsboard.server.dao.model.ModelConstants.ASSET_NAME_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.EXTERNAL_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.SEARCH_TEXT_PROPERTY; @Data @@ -68,6 +69,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity @Column(name = ModelConstants.ASSET_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = EXTERNAL_ID_PROPERTY) + private UUID externalId; + public AbstractAssetEntity() { super(); } @@ -87,6 +91,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.type = asset.getType(); this.label = asset.getLabel(); this.additionalInfo = asset.getAdditionalInfo(); + if (asset.getExternalId() != null) { + this.externalId = asset.getExternalId().getId(); + } } public AbstractAssetEntity(AssetEntity assetEntity) { @@ -99,6 +106,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.label = assetEntity.getLabel(); this.searchText = assetEntity.getSearchText(); this.additionalInfo = assetEntity.getAdditionalInfo(); + this.externalId = assetEntity.getExternalId(); } @Override @@ -128,6 +136,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity asset.setType(type); asset.setLabel(label); asset.setAdditionalInfo(additionalInfo); + if (externalId != null) { + asset.setExternalId(new AssetId(externalId)); + } return asset; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 717ba1f9e3..dbc0a057ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -84,6 +84,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti @Column(name = ModelConstants.DEVICE_DEVICE_DATA_PROPERTY, columnDefinition = "jsonb") private JsonNode deviceData; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY, columnDefinition = "uuid") + private UUID externalId; + public AbstractDeviceEntity() { super(); } @@ -113,6 +116,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.type = device.getType(); this.label = device.getLabel(); this.additionalInfo = device.getAdditionalInfo(); + if (device.getExternalId() != null) { + this.externalId = device.getExternalId().getId(); + } } public AbstractDeviceEntity(DeviceEntity deviceEntity) { @@ -129,6 +135,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.additionalInfo = deviceEntity.getAdditionalInfo(); this.firmwareId = deviceEntity.getFirmwareId(); this.softwareId = deviceEntity.getSoftwareId(); + this.externalId = deviceEntity.getExternalId(); } @Override @@ -164,6 +171,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti device.setType(type); device.setLabel(label); device.setAdditionalInfo(additionalInfo); + if (externalId != null) { + device.setExternalId(new DeviceId(externalId)); + } return device; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java index 61b21ecc4f..3dec26afd9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java @@ -77,6 +77,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Column(name = ModelConstants.CUSTOMER_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public CustomerEntity() { super(); } @@ -97,6 +100,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea this.phone = customer.getPhone(); this.email = customer.getEmail(); this.additionalInfo = customer.getAdditionalInfo(); + if (customer.getExternalId() != null) { + this.externalId = customer.getExternalId().getId(); + } } @Override @@ -124,6 +130,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea customer.setPhone(phone); customer.setEmail(email); customer.setAdditionalInfo(additionalInfo); + if (externalId != null) { + customer.setExternalId(new CustomerId(externalId)); + } return customer; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index 45b4bf9210..ccaea5524e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -78,6 +78,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Column(name = ModelConstants.DASHBOARD_CONFIGURATION_PROPERTY) private JsonNode configuration; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DashboardEntity() { super(); } @@ -102,6 +105,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S this.mobileHide = dashboard.isMobileHide(); this.mobileOrder = dashboard.getMobileOrder(); this.configuration = dashboard.getConfiguration(); + if (dashboard.getExternalId() != null) { + this.externalId = dashboard.getExternalId().getId(); + } } @Override @@ -133,6 +139,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S dashboard.setMobileHide(mobileHide); dashboard.setMobileOrder(mobileOrder); dashboard.setConfiguration(configuration); + if (externalId != null) { + dashboard.setExternalId(new DashboardId(externalId)); + } return dashboard; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java index bd214a6cbc..6894bd08d7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java @@ -103,6 +103,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl @Column(name = ModelConstants.DEVICE_PROFILE_SOFTWARE_ID_PROPERTY) private UUID softwareId; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DeviceProfileEntity() { super(); } @@ -137,6 +140,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (deviceProfile.getSoftwareId() != null) { this.softwareId = deviceProfile.getSoftwareId().getId(); } + if (deviceProfile.getExternalId() != null) { + this.externalId = deviceProfile.getExternalId().getId(); + } } @Override @@ -184,6 +190,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (softwareId != null) { deviceProfile.setSoftwareId(new OtaPackageId(softwareId)); } + if (externalId != null) { + deviceProfile.setExternalId(new DeviceProfileId(externalId)); + } return deviceProfile; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java index 09ed75b5e5..88c8c79fdc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java @@ -75,6 +75,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleChainEntity() { } @@ -94,6 +97,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT this.debugMode = ruleChain.isDebugMode(); this.configuration = ruleChain.getConfiguration(); this.additionalInfo = ruleChain.getAdditionalInfo(); + if (ruleChain.getExternalId() != null) { + this.externalId = ruleChain.getExternalId().getId(); + } } @Override @@ -120,6 +126,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT ruleChain.setDebugMode(debugMode); ruleChain.setConfiguration(configuration); ruleChain.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleChain.setExternalId(new RuleChainId(externalId)); + } return ruleChain; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 097779f33e..fd84c12a1b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -684,6 +684,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC throw t; } } + deleteRuleNodes(tenantId, ruleChainId); + } + + @Override + public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { List nodeRelations = getRuleChainToNodeRelations(tenantId, ruleChainId); for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 542a487949..59f287eadd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -19,19 +19,18 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Collection; -import java.util.List; import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. @@ -81,4 +80,6 @@ public interface RuleChainDao extends Dao, TenantEntityDao { Collection findByTenantIdAndTypeAndName(TenantId tenantId, RuleChainType type, String name); + RuleChain findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 833fcc2de8..140ae3af86 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -public interface AssetRepository extends JpaRepository { +public interface AssetRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo) " + "FROM AssetEntity a " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index c9752c5390..cd4fefac48 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -208,4 +208,20 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); } + + @Override + public Asset findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Asset findByTenantIdAndId(UUID tenantId, UUID id) { + return DaoUtil.getData(assetRepository.findByTenantIdAndId(tenantId, id)); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 61cd9afac7..1300fcdfa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.CustomerEntity; import java.util.UUID; @@ -27,7 +28,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface CustomerRepository extends JpaRepository { +public interface CustomerRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT c FROM CustomerEntity c WHERE c.tenantId = :tenantId " + "AND LOWER(c.searchText) LIKE LOWER(CONCAT('%', :textSearch, '%'))") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 02311df61c..9cf74f1f33 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -19,6 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -68,4 +69,20 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao { +public interface DashboardRepository extends JpaRepository, ExportableEntityRepository { Long countByTenantId(UUID tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 98edcaa1fd..b65e9997a9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -19,7 +19,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; @@ -49,4 +51,20 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao { +public interface DeviceProfileRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.common.data.DeviceProfileInfo(d.id, d.name, d.image, d.defaultDashboardId, d.type, d.transportType) " + "FROM DeviceProfileEntity d " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index b4ae9270e3..46d755195e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -21,6 +21,7 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -30,7 +31,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface DeviceRepository extends JpaRepository { +public interface DeviceRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.DeviceInfoEntity(d, c.title, c.additionalInfo, p.name) " + "FROM DeviceEntity d " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 1d75690e50..1a5d44900c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -302,4 +302,20 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao Objects.toString(pageLink.getTextSearch(), ""), DaoUtil.toPageable(pageLink))); } + + @Override + public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Device findByTenantIdAndId(UUID tenantId, UUID id) { + return findDeviceByTenantIdAndId(TenantId.fromUUID(tenantId), id); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 80513d77c6..539377a647 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -23,6 +23,7 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -109,4 +110,20 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao { +public interface RuleChainRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT rc FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId " + "AND LOWER(rc.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 4e2596485b..e18f73a065 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -72,17 +72,19 @@ CREATE TABLE IF NOT EXISTS entity_alarm ( CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS asset ( - id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - type varchar(255), - CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) +CREATE TABLE IF NOT EXISTS asset( + id uuid NOT NULL + CONSTRAINT asset_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + type varchar(255), + external_id uuid, + CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) ); CREATE TABLE IF NOT EXISTS audit_log ( @@ -128,47 +130,53 @@ CREATE TABLE IF NOT EXISTS component_descriptor ( ); CREATE TABLE IF NOT EXISTS customer ( - id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - address varchar, - address2 varchar, - city varchar(255), - country varchar(255), - email varchar(255), - phone varchar(255), - search_text varchar(255), - state varchar(255), - tenant_id uuid, - title varchar(255), - zip varchar(255) -); - -CREATE TABLE IF NOT EXISTS dashboard ( - id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY, - created_time bigint NOT NULL, - configuration varchar, + id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + address varchar, + address2 varchar, + city varchar(255), + country varchar(255), + email varchar(255), + phone varchar(255), + search_text varchar(255), + state varchar(255), + tenant_id uuid, + title varchar(255), + zip varchar(255), + external_id uuid +); + +CREATE TABLE IF NOT EXISTS dashboard +( + id uuid NOT NULL + CONSTRAINT dashboard_pkey PRIMARY KEY, + created_time bigint NOT NULL, + configuration varchar, assigned_customers varchar(1000000), - search_text varchar(255), - tenant_id uuid, - title varchar(255), - mobile_hide boolean DEFAULT false, - mobile_order int, - image varchar(1000000) + search_text varchar(255), + tenant_id uuid, + title varchar(255), + mobile_hide boolean DEFAULT false, + mobile_order int, + image varchar(1000000), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_chain ( - id uuid NOT NULL CONSTRAINT rule_chain_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - configuration varchar(10000000), - name varchar(255), - type varchar(255), - first_rule_node_id uuid, - root boolean, - debug_mode boolean, - search_text varchar(255), - tenant_id uuid + id uuid NOT NULL + CONSTRAINT rule_chain_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + configuration varchar(10000000), + name varchar(255), + type varchar(255), + first_rule_node_id uuid, + root boolean, + debug_mode boolean, + search_text varchar(255), + tenant_id uuid, + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node ( @@ -216,30 +224,31 @@ CREATE TABLE IF NOT EXISTS ota_package ( ); CREATE TABLE IF NOT EXISTS device_profile ( - id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, - created_time bigint NOT NULL, - name varchar(255), - type varchar(255), - image varchar(1000000), - transport_type varchar(255), - provision_type varchar(255), - profile_data jsonb, - description varchar, - search_text varchar(255), - is_default boolean, - tenant_id uuid, - firmware_id uuid, - software_id uuid, - default_rule_chain_id uuid, - default_dashboard_id uuid, - default_queue_name varchar(255), - provision_device_key varchar, - CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), - CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), - CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id), - CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package(id), - CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) + id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, + created_time bigint NOT NULL, + name varchar(255), + type varchar(255), + image varchar(1000000), + transport_type varchar(255), + provision_type varchar(255), + profile_data jsonb, + description varchar, + search_text varchar(255), + is_default boolean, + tenant_id uuid, + firmware_id uuid, + software_id uuid, + default_rule_chain_id uuid, + default_dashboard_id uuid, + default_queue_name varchar(255), + provision_device_key varchar, + external_id uuid, + CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), + CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain (id), + CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard (id), + CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package (id), + CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package (id) ); ALTER TABLE ota_package @@ -257,23 +266,24 @@ ALTER TABLE ota_package -- ); CREATE TABLE IF NOT EXISTS device ( - id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - device_profile_id uuid NOT NULL, - device_data jsonb, - type varchar(255), - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - firmware_id uuid, - software_id uuid, - CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), - CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), - CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package(id) + id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + device_profile_id uuid NOT NULL, + device_data jsonb, + type varchar(255), + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + firmware_id uuid, + software_id uuid, + external_id uuid, + CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile (id), + CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package (id), + CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package (id) ); CREATE TABLE IF NOT EXISTS device_credentials ( From 849513541ecb7cab4e09a1bdf3cf29480bd28ea0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 17:35:34 +0200 Subject: [PATCH 002/156] Entities export/import API refactoring: permission checks and entity lifecycle events --- .../server/controller/AssetController.java | 18 +--- .../server/controller/BaseController.java | 62 ++++++++++++++ .../server/controller/CustomerController.java | 10 +-- .../controller/DashboardController.java | 8 +- .../server/controller/DeviceController.java | 22 +---- .../controller/DeviceProfileController.java | 25 +----- .../EntitiesExportImportController.java | 84 ++++++++++++------- .../controller/RuleChainController.java | 18 +--- .../DefaultEntitiesExportImportService.java | 13 ++- .../expimp/EntitiesExportImportService.java | 5 +- .../expimp/imp/EntityImportResult.java | 26 ++++++ .../expimp/imp/EntityImportService.java | 3 +- .../imp/impl/AbstractEntityImportService.java | 3 +- .../expimp/imp/impl/AssetImportService.java | 8 +- .../imp/impl/CustomerImportService.java | 8 +- .../imp/impl/DashboardImportService.java | 8 +- .../expimp/imp/impl/DeviceImportService.java | 10 ++- .../imp/impl/DeviceProfileImportService.java | 8 +- .../imp/impl/RuleChainImportService.java | 8 +- .../common/data/export/EntityExportData.java | 5 +- .../data/export/impl/AssetExportData.java | 5 ++ .../data/export/impl/CustomerExportData.java | 5 ++ .../data/export/impl/DashboardExportData.java | 5 ++ .../data/export/impl/DeviceExportData.java | 5 ++ .../export/impl/DeviceProfileExportData.java | 5 ++ .../data/export/impl/RuleChainExportData.java | 5 ++ 26 files changed, 240 insertions(+), 142 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 7bd9cb065d..94c714e24f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -156,7 +156,7 @@ public class AssetController extends BaseController { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - onAssetCreatedOrUpdated(savedAsset, asset.getId() != null, getCurrentUser()); + onEntityUpdatedOrCreated(getCurrentUser(), savedAsset, null, asset.getId() == null); return savedAsset; } catch (Exception e) { @@ -166,20 +166,6 @@ public class AssetController extends BaseController { } } - private void onAssetCreatedOrUpdated(Asset asset, boolean updated, SecurityUser user) { - try { - logEntityAction(user, asset.getId(), asset, - asset.getCustomerId(), - updated ? ActionType.UPDATED : ActionType.ADDED, null); - } catch (ThingsboardException e) { - log.error("Failed to log entity action", e); - } - - if (updated) { - sendEntityNotificationMsg(asset.getTenantId(), asset.getId(), EdgeEventActionType.UPDATED); - } - } - @ApiOperation(value = "Delete asset (deleteAsset)", notes = "Deletes the asset and all the relations (from and to the asset). Referencing non-existing asset Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -681,7 +667,7 @@ public class AssetController extends BaseController { public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { - onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated(), user); + onEntityUpdatedOrCreated(user, importedAssetInfo.getEntity(), importedAssetInfo.getOldEntity(), !importedAssetInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index f945cc30b7..210a6bea2d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.OtaPackage; @@ -68,6 +69,7 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -83,6 +85,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rpc.Rpc; @@ -141,6 +144,7 @@ import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -919,4 +923,62 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + public & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + boolean notifyEdge = false; + + EntityType entityType = savedEntity.getId().getEntityType(); + switch (entityType) { + case DEVICE: + tbClusterService.onDeviceUpdated((Device) savedEntity, (Device) oldEntity); + break; + case DEVICE_PROFILE: + DeviceProfile deviceProfile = (DeviceProfile) savedEntity; + DeviceProfile oldDeviceProfile = (DeviceProfile) oldEntity; + boolean isFirmwareChanged = false; + boolean isSoftwareChanged = false; + if (!isNewEntity) { + if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { + isFirmwareChanged = true; + } + if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { + isSoftwareChanged = true; + } + } + tbClusterService.onDeviceProfileChange(deviceProfile, null); + tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), deviceProfile.getId(), + isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); + notifyEdge = true; + break; + case RULE_CHAIN: // FIXME: events for rule chain metadata + RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); + if (RuleChainType.CORE.equals(ruleChainType)) { + tbClusterService.broadcastEntityStateChangeEvent(savedEntity.getTenantId(), savedEntity.getId(), + isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + if (RuleChainType.EDGE.equals(ruleChainType)) { + if (!isNewEntity) { + notifyEdge = true; + } + } + break; + case ASSET: + case CUSTOMER: + case DASHBOARD: + if (!isNewEntity) { + notifyEdge = true; + } + break; + default: + throw new UnsupportedOperationException(); + } + + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, + isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (notifyEdge) { + sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index b419121459..27037f36e7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -33,7 +33,6 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; @@ -151,14 +150,7 @@ public class CustomerController extends BaseController { checkEntity(customer.getId(), customer, Resource.CUSTOMER); Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - - logEntityAction(savedCustomer.getId(), savedCustomer, - savedCustomer.getId(), - customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); - - if (customer.getId() != null) { - sendEntityNotificationMsg(savedCustomer.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); - } + onEntityUpdatedOrCreated(getCurrentUser(), savedCustomer, null, customer.getId() == null); return savedCustomer; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 49c33045a5..4ea1ca83c7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -187,13 +187,7 @@ public class DashboardController extends BaseController { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - logEntityAction(savedDashboard.getId(), savedDashboard, - null, - dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); - - if (dashboard.getId() != null) { - sendEntityNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); - } + onEntityUpdatedOrCreated(getCurrentUser(), savedDashboard, null, dashboard.getId() == null); return savedDashboard; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 85a21e912c..d2ffafa73e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -195,7 +195,7 @@ public class DeviceController extends BaseController { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created, getCurrentUser()); + onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, oldDevice, created); return savedDevice; } catch (Exception e) { @@ -224,10 +224,8 @@ public class DeviceController extends BaseController { checkEntity(device.getId(), device, Resource.DEVICE); Device savedDevice = deviceService.saveDeviceWithCredentials(device, credentials); checkNotNull(savedDevice); - tbClusterService.onDeviceUpdated(savedDevice, device); - logEntityAction(savedDevice.getId(), savedDevice, - savedDevice.getCustomerId(), - device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + + onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, device, device.getId() == null); return savedDevice; } catch (Exception e) { @@ -237,18 +235,6 @@ public class DeviceController extends BaseController { } } - private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated, SecurityUser user) { - tbClusterService.onDeviceUpdated(savedDevice, oldDevice); - - try { - logEntityAction(user, savedDevice.getId(), savedDevice, - savedDevice.getCustomerId(), - updated ? ActionType.UPDATED : ActionType.ADDED, null); - } catch (ThingsboardException e) { - log.error("Failed to log entity action", e); - } - } - @ApiOperation(value = "Delete device (deleteDevice)", notes = "Deletes the device, it's credentials and all the relations (from and to the device). Referencing non-existing device Id will cause an error." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -1015,7 +1001,7 @@ public class DeviceController extends BaseController { public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { - onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated(), user); + onEntityUpdatedOrCreated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java index 25a35e77fa..240e60be2c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java @@ -46,7 +46,6 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; -import java.util.Objects; import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_DATA; @@ -207,32 +206,14 @@ public class DeviceProfileController extends BaseController { checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE); - boolean isFirmwareChanged = false; - boolean isSoftwareChanged = false; - + DeviceProfile oldDeviceProfile = null; if (!created) { - DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); - if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { - isFirmwareChanged = true; - } - if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { - isSoftwareChanged = true; - } + oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); - tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), savedDeviceProfile.getId(), - created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - - logEntityAction(savedDeviceProfile.getId(), savedDeviceProfile, - null, - created ? ActionType.ADDED : ActionType.UPDATED, null); - - otaPackageStateService.update(savedDeviceProfile, isFirmwareChanged, isSoftwareChanged); + onEntityUpdatedOrCreated(getCurrentUser(), deviceProfile, oldDeviceProfile, created); - sendEntityNotificationMsg(getTenantId(), savedDeviceProfile.getId(), - deviceProfile.getId() == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); return savedDeviceProfile; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile, diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index a11a9189f4..07a033ab03 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; @@ -23,21 +24,20 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.export.EntitiesExportRequest; -import org.thingsboard.server.common.data.export.EntitiesExportResponse; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.EntitiesExportImportService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; import java.util.UUID; @RestController @@ -51,49 +51,69 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@PathVariable EntityType entityType, + public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") @PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); try { - return exportImportService.exportEntity(getTenantId(), entityId); + return exportEntity(getCurrentUser(), entityId); } catch (Exception e) { throw handleException(e); } } - @PostMapping("/export/batch") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { - TenantId tenantId = getTenantId(); - - EntitiesExportResponse exportResponse = new EntitiesExportResponse(); - - Map>>> result = new HashMap<>(); - exportRequest.getEntities().forEach((entityType, entityIds) -> { - List>> exportDataForEntityType = new LinkedList<>(); - entityIds.forEach(entityId -> { - EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); - exportDataForEntityType.add(exportData); - }); - result.put(entityType, exportDataForEntityType); - }); - - exportResponse.setExportData(result); - return exportResponse; - } + +// @PostMapping("/export/batch") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { +// TenantId tenantId = getTenantId(); +// +// EntitiesExportResponse exportResponse = new EntitiesExportResponse(); +// +// Map>>> result = new HashMap<>(); +// exportRequest.getEntities().forEach((entityType, entityIds) -> { +// List>> exportDataForEntityType = new LinkedList<>(); +// entityIds.forEach(entityId -> { +// EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); +// exportDataForEntityType.add(exportData); +// }); +// result.put(entityType, exportDataForEntityType); +// }); +// +// exportResponse.setExportData(result); +// return exportResponse; +// } + // TODO: export and import of batches // TODO: api to export and import whole customer, whole tenant @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public , I extends EntityId, D extends EntityExportData> E importEntity(@RequestBody D exportData) throws ThingsboardException { + public & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(@RequestBody D exportData) throws ThingsboardException { try { - return exportImportService.importEntity(getTenantId(), exportData); + return importEntity(getCurrentUser(), exportData); } catch (Exception e) { throw handleException(e); } } -// public void importEntities(@RequestBody ) + + private , I extends EntityId> EntityExportData> exportEntity(SecurityUser user, I entityId) throws ThingsboardException { + checkEntityId(entityId, Operation.READ); + return exportImportService.exportEntity(getTenantId(), entityId); + } + + private & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(SecurityUser user, D exportData) throws ThingsboardException { + E existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { + checkEntityId(existingEntity.getId(), Operation.WRITE); // todo maybe need to extract permission check to BaseController and put there permission checks from other controllers + } else { + checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); + } + + EntityImportResult importResult = exportImportService.importEntity(getTenantId(), exportData); + onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); + + return importResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 6479b838b2..aba074ac60 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -78,11 +78,9 @@ import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -254,20 +252,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - if (RuleChainType.CORE.equals(savedRuleChain.getType())) { - tbClusterService.broadcastEntityStateChangeEvent(ruleChain.getTenantId(), savedRuleChain.getId(), - created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } - - logEntityAction(savedRuleChain.getId(), savedRuleChain, - null, - created ? ActionType.ADDED : ActionType.UPDATED, null); - - if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { - if (!created) { - sendEntityNotificationMsg(savedRuleChain.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); - } - } + onEntityUpdatedOrCreated(getCurrentUser(), savedRuleChain, null, created); return savedRuleChain; } catch (Exception e) { @@ -294,6 +279,7 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); + tbClusterService.broadcastEntityStateChangeEvent(savedRuleChain.getTenantId(), savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, null, ActionType.ADDED, null); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index 9b384b34dd..26d983a694 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -24,7 +24,9 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; +import org.thingsboard.server.service.expimp.imp.impl.AbstractEntityImportService; import java.util.Collection; import java.util.EnumMap; @@ -51,16 +53,21 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } // FIXME: somehow validate export data - // FIXME: validate permissions for create or update - // FIXME: send entity lifecycle event @Override - public , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData) { + public , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData) { EntityType entityType = exportData.getEntityType(); EntityImportService importService = getImportService(entityType); return importService.importEntity(tenantId, exportData); } + @Override + @SuppressWarnings("unchecked") + public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { + return (E) importServices.values().stream().filter(entityImportService -> entityImportService instanceof AbstractEntityImportService) + .findFirst().map(entityImportService -> (AbstractEntityImportService) importServices).get() + .findByExternalOrInternalId(tenantId, externalId); // FIXME !!! + } @SuppressWarnings("unchecked") private > EntityExportService getExportService(EntityType entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 5d25cb9320..75503d2750 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -19,11 +19,14 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); - , I extends EntityId, D extends EntityExportData> E importEntity(TenantId tenantId, D exportData); + , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData); + + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java new file mode 100644 index 0000000000..9868670076 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.imp; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; + +@Data +public class EntityImportResult> { + private E savedEntity; + private E oldEntity; +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java index 195ace0b66..e0f469e751 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -23,7 +23,8 @@ import org.thingsboard.server.common.data.id.TenantId; public interface EntityImportService, D extends EntityExportData> { - E importEntity(TenantId tenantId, D exportData); + // FIXME: get rid of boilerplate for import result creation and everything else + EntityImportResult importEntity(TenantId tenantId, D exportData); EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index 3f33ba04f0..81855ec8dd 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -38,7 +38,6 @@ public abstract class AbstractEntityImportService ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; @@ -50,7 +49,7 @@ public abstract class AbstractEntityImportService, ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { + public final , ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { ExportableEntityDao dao = getDao(externalOrInternalId.getEntityType()); return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalOrInternalId.getId())) .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalOrInternalId.getId())); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java index 5e563762d6..0e3c6f3004 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class AssetImportService extends AbstractEntityImportService importEntity(TenantId tenantId, AssetExportData exportData) { Asset asset = exportData.getAsset(); Asset existingAsset = findByExternalId(tenantId, asset.getId()); // TODO: extract boiler plate to abstract class ... @@ -51,7 +52,10 @@ public class AssetImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedAsset); + importResult.setOldEntity(existingAsset); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java index 9510cc0700..5f98a7fb66 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class CustomerImportService extends AbstractEntityImportService importEntity(TenantId tenantId, CustomerExportData exportData) { Customer customer = exportData.getCustomer(); Customer existingCustomer = findByExternalId(tenantId, customer.getId()); @@ -49,7 +50,10 @@ public class CustomerImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedCustomer); + importResult.setOldEntity(existingCustomer); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java index eab90bb294..210e7ca841 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class DashboardImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DashboardExportData exportData) { Dashboard dashboard = exportData.getDashboard(); Dashboard existingDashboard = findByExternalId(tenantId, dashboard.getId()); @@ -51,7 +52,10 @@ public class DashboardImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedDashboard); + importResult.setOldEntity(existingDashboard); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java index 052fa3e1b3..c72ed48b2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.expimp.imp.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.impl.DeviceExportData; @@ -26,7 +25,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -38,7 +37,7 @@ public class DeviceImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceExportData exportData) { Device device = exportData.getDevice(); Device existingDevice = findByExternalId(tenantId, device.getId()); // FIXME: !!! // what if exporting and importing back already exported entity ? (save version and then load it back) @@ -70,7 +69,10 @@ public class DeviceImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedDevice); + importResult.setOldEntity(existingDevice); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java index 615f62d389..99efefe812 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class DeviceProfileImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceProfileExportData exportData) { DeviceProfile deviceProfile = exportData.getDeviceProfile(); DeviceProfile existingDeviceProfile = findByExternalId(tenantId, deviceProfile.getId()); @@ -54,7 +55,10 @@ public class DeviceProfileImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedDeviceProfile); + importResult.setOldEntity(existingDeviceProfile); + return importResult; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java index eb72f58a7f..92daa19a00 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -37,7 +38,7 @@ public class RuleChainImportService extends AbstractEntityImportService importEntity(TenantId tenantId, RuleChainExportData exportData) { RuleChain ruleChain = exportData.getRuleChain(); RuleChain existingRuleChain = findByExternalId(tenantId, ruleChain.getId()); @@ -69,7 +70,10 @@ public class RuleChainImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedRuleChain); + importResult.setOldEntity(existingRuleChain); + return importResult; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java index 36d3cb4b0c..4fd1a5c732 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -43,6 +43,9 @@ import org.thingsboard.server.common.data.id.HasId; public interface EntityExportData> { @JsonIgnore - EntityType getEntityType(); // fixme: maybe remove if not needed, as well as generic + E getMainEntity(); + + @JsonIgnore + EntityType getEntityType(); // fixme: maybe remove if not needed } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java index c0f417bd76..abb268eeb6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java @@ -25,6 +25,11 @@ public class AssetExportData implements EntityExportData { private Asset asset; + @Override + public Asset getMainEntity() { + return asset; + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java index 31e53db223..02ec9e7efa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java @@ -25,6 +25,11 @@ public class CustomerExportData implements EntityExportData { private Customer customer; + @Override + public Customer getMainEntity() { + return customer; + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java index 7860c6c553..f75f2479f6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java @@ -25,6 +25,11 @@ public class DashboardExportData implements EntityExportData { private Dashboard dashboard; + @Override + public Dashboard getMainEntity() { + return dashboard; + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java index 05f3299d02..ff3741dabc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java @@ -27,6 +27,11 @@ public class DeviceExportData implements EntityExportData { private Device device; private DeviceCredentials credentials; + @Override + public Device getMainEntity() { + return device; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java index c030d0b86e..a5d7872cc6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java @@ -25,6 +25,11 @@ public class DeviceProfileExportData implements EntityExportData private DeviceProfile deviceProfile; + @Override + public DeviceProfile getMainEntity() { + return deviceProfile; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java index 11fbace911..101f516881 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java @@ -27,6 +27,11 @@ public class RuleChainExportData implements EntityExportData { private RuleChain ruleChain; private RuleChainMetaData metaData; + @Override + public RuleChain getMainEntity() { + return ruleChain; + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; From 99d0197caa4dc0c31d3ac726b88cec33c96d2bff Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 18:01:44 +0200 Subject: [PATCH 003/156] Refactor AbstractEntityImportService --- .../server/controller/BaseController.java | 10 ++--- .../DefaultEntitiesExportImportService.java | 32 +++++++++++----- .../expimp/EntitiesExportImportService.java | 4 +- .../expimp/ExportableEntitiesService.java | 26 +++++++++++++ .../imp/impl/AbstractEntityImportService.java | 38 ++++--------------- 5 files changed, 61 insertions(+), 49 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 210a6bea2d..e244bcf680 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -925,7 +925,7 @@ public abstract class BaseController { } public & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { - boolean notifyEdge = false; + boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); switch (entityType) { @@ -949,7 +949,7 @@ public abstract class BaseController { tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), deviceProfile.getId(), isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); - notifyEdge = true; + notifyEdgeService = true; break; case RULE_CHAIN: // FIXME: events for rule chain metadata RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); @@ -959,7 +959,7 @@ public abstract class BaseController { } if (RuleChainType.EDGE.equals(ruleChainType)) { if (!isNewEntity) { - notifyEdge = true; + notifyEdgeService = true; } } break; @@ -967,7 +967,7 @@ public abstract class BaseController { case CUSTOMER: case DASHBOARD: if (!isNewEntity) { - notifyEdge = true; + notifyEdgeService = true; } break; default: @@ -976,7 +976,7 @@ public abstract class BaseController { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (notifyEdge) { + if (notifyEdgeService) { sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); } } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index 26d983a694..352d26d177 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -22,23 +22,25 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.exp.EntityExportService; import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; -import org.thingsboard.server.service.expimp.imp.impl.AbstractEntityImportService; import java.util.Collection; -import java.util.EnumMap; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; // FIXME: review packages and classes naming @Service @TbCoreComponent public class DefaultEntitiesExportImportService implements EntitiesExportImportService { - private final Map> exportServices = new EnumMap<>(EntityType.class); - private final Map> importServices = new EnumMap<>(EntityType.class); + private final Map> exportServices = new HashMap<>(); + private final Map> importServices = new HashMap<>(); + private final Map> daos = new HashMap<>(); // TODO: export and import of the whole tenant @@ -61,14 +63,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importService.importEntity(tenantId, exportData); } + @Override - @SuppressWarnings("unchecked") public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { - return (E) importServices.values().stream().filter(entityImportService -> entityImportService instanceof AbstractEntityImportService) - .findFirst().map(entityImportService -> (AbstractEntityImportService) importServices).get() - .findByExternalOrInternalId(tenantId, externalId); // FIXME !!! + ExportableEntityDao dao = getDao(externalId.getEntityType()); + return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) + .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalId.getId())); } + @SuppressWarnings("unchecked") private > EntityExportService getExportService(EntityType entityType) { return (EntityExportService) exportServices.get(entityType); @@ -79,15 +82,24 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return (EntityImportService) importServices.get(entityType); } + @SuppressWarnings("unchecked") + private ExportableEntityDao getDao(EntityType entityType) { + return (ExportableEntityDao) daos.get(entityType); + } + @Autowired - private void setExportImportServices(Collection> exportServices, - Collection> importServices) { + private void setServices(Collection> exportServices, + Collection> importServices, + Collection> daos) { exportServices.forEach(entityExportService -> { this.exportServices.put(entityExportService.getEntityType(), entityExportService); }); importServices.forEach(entityImportService -> { this.importServices.put(entityImportService.getEntityType(), entityImportService); }); + daos.forEach(dao -> { + this.daos.put(dao.getEntityType(), dao); + }); } } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 75503d2750..2a2ac3e1f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -21,12 +21,10 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.expimp.imp.EntityImportResult; -public interface EntitiesExportImportService { +public interface EntitiesExportImportService extends ExportableEntitiesService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData); - , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); - } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java new file mode 100644 index 0000000000..0af93dfeec --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ExportableEntitiesService { + + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index 81855ec8dd..795af18db3 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -16,57 +16,33 @@ package org.thingsboard.server.service.expimp.imp.impl; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.server.common.data.EntityType; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.service.expimp.ExportableEntitiesService; import org.thingsboard.server.service.expimp.imp.EntityImportService; -import java.util.Collection; -import java.util.EnumMap; -import java.util.Map; -import java.util.Optional; - public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { - private final Map> daos = new EnumMap<>(EntityType.class); + @Autowired @Lazy + private ExportableEntitiesService exportableEntitiesService; - public final E findByExternalId(TenantId tenantId, I externalId) { - return findByExternalOrInternalId(tenantId, externalId); + protected final E findByExternalId(TenantId tenantId, I externalId) { + return exportableEntitiesService.findEntityByExternalId(tenantId, externalId); } protected final ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; } - HasId entity = findByExternalOrInternalId(tenantId, externalId); + HasId entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); if (entity == null) { throw new IllegalStateException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); } return entity.getId(); } - public final , ID extends EntityId> T findByExternalOrInternalId(TenantId tenantId, ID externalOrInternalId) { - ExportableEntityDao dao = getDao(externalOrInternalId.getEntityType()); - return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalOrInternalId.getId())) - .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalOrInternalId.getId())); - } - - - @SuppressWarnings("unchecked") - private ExportableEntityDao getDao(EntityType entityType) { - return (ExportableEntityDao) daos.get(entityType); - } - - @Autowired - private void setDaos(Collection> daos) { - daos.forEach(dao -> this.daos.put(dao.getEntityType(), dao)); - if (!this.daos.containsKey(getEntityType())) { - throw new IllegalStateException(getClass().getSimpleName() + " requires ExportableEntityDao for entity type " + getEntityType()); - } - } - } From 01ad35d8645f51b763a1d3acf2c6cc72f8188bd1 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 16 Mar 2022 20:02:51 +0200 Subject: [PATCH 004/156] Get rid of boilerplate in import services --- .../EntitiesExportImportController.java | 14 +++---- .../DefaultEntitiesExportImportService.java | 14 +++---- .../expimp/EntitiesExportImportService.java | 6 +-- .../expimp/ExportableEntitiesService.java | 4 +- .../expimp/exp/EntityExportService.java | 4 +- .../expimp/imp/EntityImportResult.java | 4 +- .../expimp/imp/EntityImportService.java | 5 +-- .../imp/impl/AbstractEntityImportService.java | 40 ++++++++++++++++++- .../expimp/imp/impl/AssetImportService.java | 22 +--------- .../imp/impl/CustomerImportService.java | 22 +--------- .../imp/impl/DashboardImportService.java | 20 ++-------- .../expimp/imp/impl/DeviceImportService.java | 38 ++---------------- .../imp/impl/DeviceProfileImportService.java | 22 +--------- .../imp/impl/RuleChainImportService.java | 27 +++---------- .../server/common/data/Customer.java | 3 +- .../server/common/data/Dashboard.java | 3 +- .../server/common/data/Device.java | 3 +- .../server/common/data/DeviceProfile.java | 3 +- .../server/common/data/asset/Asset.java | 3 +- .../data/export/EntitiesExportResponse.java | 3 +- .../common/data/export/EntityExportData.java | 3 +- .../common/data/export/ExportableEntity.java | 39 ++++++++++++++++++ .../server/common/data/rule/RuleChain.java | 3 +- 23 files changed, 133 insertions(+), 172 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 07a033ab03..a3f26bf7b0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -24,13 +24,11 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.EntitiesExportImportService; import org.thingsboard.server.service.expimp.imp.EntityImportResult; @@ -88,7 +86,7 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(@RequestBody D exportData) throws ThingsboardException { + public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData) throws ThingsboardException { try { return importEntity(getCurrentUser(), exportData); } catch (Exception e) { @@ -97,20 +95,20 @@ public class EntitiesExportImportController extends BaseController { } - private , I extends EntityId> EntityExportData> exportEntity(SecurityUser user, I entityId) throws ThingsboardException { + private EntityExportData> exportEntity(SecurityUser user, EntityId entityId) throws ThingsboardException { checkEntityId(entityId, Operation.READ); return exportImportService.exportEntity(getTenantId(), entityId); } - private & HasName & HasTenantId, I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(SecurityUser user, D exportData) throws ThingsboardException { - E existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData) throws ThingsboardException { + ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); if (existingEntity != null) { checkEntityId(existingEntity.getId(), Operation.WRITE); // todo maybe need to extract permission check to BaseController and put there permission checks from other controllers } else { checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } - EntityImportResult importResult = exportImportService.importEntity(getTenantId(), exportData); + EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData); onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); return importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index 352d26d177..c6fe03a750 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -19,8 +19,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -47,7 +47,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS // TODO: export and import of the whole customer ? // TODO: relations export and import @Override - public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { + public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { EntityType entityType = entityId.getEntityType(); EntityExportService exportService = getExportService(entityType); @@ -56,16 +56,16 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS // FIXME: somehow validate export data @Override - public , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData) { + public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData) { EntityType entityType = exportData.getEntityType(); - EntityImportService importService = getImportService(entityType); + EntityImportService> importService = getImportService(entityType); return importService.importEntity(tenantId, exportData); } @Override - public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { + public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { ExportableEntityDao dao = getDao(externalId.getEntityType()); return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalId.getId())); @@ -73,12 +73,12 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @SuppressWarnings("unchecked") - private > EntityExportService getExportService(EntityType entityType) { + private > EntityExportService getExportService(EntityType entityType) { return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") - private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { return (EntityImportService) importServices.get(entityType); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 2a2ac3e1f5..10361865a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -16,15 +16,15 @@ package org.thingsboard.server.service.expimp; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.expimp.imp.EntityImportResult; public interface EntitiesExportImportService extends ExportableEntitiesService { - , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); + , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); - , I extends EntityId, D extends EntityExportData> EntityImportResult importEntity(TenantId tenantId, D exportData); + , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java index 0af93dfeec..ca401db64b 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java @@ -15,12 +15,12 @@ */ package org.thingsboard.server.service.expimp; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; public interface ExportableEntitiesService { - , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java index 9afa01ff8e..bdd525d5d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java @@ -17,11 +17,11 @@ package org.thingsboard.server.service.expimp.exp; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntityExportService> { +public interface EntityExportService> { // FIXME: export relations // FIXME: get rid of boilerplate diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java index 9868670076..90f5d42bc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java @@ -16,11 +16,11 @@ package org.thingsboard.server.service.expimp.imp; import lombok.Data; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; @Data -public class EntityImportResult> { +public class EntityImportResult> { private E savedEntity; private E oldEntity; } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java index e0f469e751..96a0f732ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -17,13 +17,12 @@ package org.thingsboard.server.service.expimp.imp; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntityImportService, D extends EntityExportData> { +public interface EntityImportService, D extends EntityExportData> { - // FIXME: get rid of boilerplate for import result creation and everything else EntityImportResult importEntity(TenantId tenantId, D exportData); EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index 795af18db3..fd635e239c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -18,22 +18,58 @@ package org.thingsboard.server.service.expimp.imp.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.expimp.ExportableEntitiesService; +import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; -public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { +public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; + // FIXME what if exporting and importing back already exported entity ? (save version and then load it back) + /* + * export entity -> id from env1 -> import this entity -> ... + * + * maybe find not only by external id but by internal too ? but then what if we will try + * */ + @Override + public final EntityImportResult importEntity(TenantId tenantId, D exportData) { + E entity = exportData.getMainEntity(); + E existingEntity = findByExternalId(tenantId, entity.getId()); - protected final E findByExternalId(TenantId tenantId, I externalId) { + entity.setExternalId(entity.getId()); + entity.setTenantId(tenantId); + + if (existingEntity == null) { + entity.setId(null); + } else { + entity.setId(existingEntity.getId()); + } + + E savedEntity = prepareAndSaveEntity(tenantId, entity, existingEntity, exportData); + + EntityImportResult importResult = new EntityImportResult<>(); + importResult.setSavedEntity(savedEntity); + importResult.setOldEntity(existingEntity); + return importResult; + } + + protected abstract E prepareAndSaveEntity(TenantId tenantId, E entity, E existingEntity, D exportData); + + + private E findByExternalId(TenantId tenantId, I externalId) { return exportableEntitiesService.findEntityByExternalId(tenantId, externalId); } + // TODO or maybe set as additional config whether to update related entities when device already exists ? + // TODO: or also whether to ignore not found internal ids + + // FIXME: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants protected final ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java index 0e3c6f3004..4e6fa2b33e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,27 +34,10 @@ public class AssetImportService extends AbstractEntityImportService importEntity(TenantId tenantId, AssetExportData exportData) { - Asset asset = exportData.getAsset(); - Asset existingAsset = findByExternalId(tenantId, asset.getId()); // TODO: extract boiler plate to abstract class ... - - asset.setExternalId(asset.getId()); - asset.setTenantId(tenantId); - - if (existingAsset == null) { - asset.setId(null); - } else { - asset.setId(existingAsset.getId()); - } - + protected Asset prepareAndSaveEntity(TenantId tenantId, Asset asset, Asset existingAsset, AssetExportData exportData) { asset.setCustomerId(getInternalId(tenantId, asset.getCustomerId())); - Asset savedAsset = assetService.saveAsset(asset); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedAsset); - importResult.setOldEntity(existingAsset); - return importResult; + return assetService.saveAsset(asset); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java index 5f98a7fb66..559cc1cd18 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,25 +34,8 @@ public class CustomerImportService extends AbstractEntityImportService importEntity(TenantId tenantId, CustomerExportData exportData) { - Customer customer = exportData.getCustomer(); - Customer existingCustomer = findByExternalId(tenantId, customer.getId()); - - customer.setExternalId(customer.getId()); - customer.setTenantId(tenantId); - - if (existingCustomer == null) { - customer.setId(null); - } else { - customer.setId(existingCustomer.getId()); - } - - Customer savedCustomer = customerService.saveCustomer(customer); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedCustomer); - importResult.setOldEntity(existingCustomer); - return importResult; + protected Customer prepareAndSaveEntity(TenantId tenantId, Customer customer, Customer existingCustomer, CustomerExportData exportData) { + return customerService.saveCustomer(customer); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java index 210e7ca841..916d95ea1c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,27 +34,14 @@ public class DashboardImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DashboardExportData exportData) { - Dashboard dashboard = exportData.getDashboard(); - Dashboard existingDashboard = findByExternalId(tenantId, dashboard.getId()); - - dashboard.setExternalId(dashboard.getId()); - dashboard.setTenantId(tenantId); - + protected Dashboard prepareAndSaveEntity(TenantId tenantId, Dashboard dashboard, Dashboard existingDashboard, DashboardExportData exportData) { if (existingDashboard == null) { - dashboard.setId(null); dashboard.setAssignedCustomers(null); // FIXME: need to assign dashboard to customers ? } else { - dashboard.setId(existingDashboard.getId()); - dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); // we left them untouched (FIXME) + dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); } - Dashboard savedDashboard = dashboardService.saveDashboard(dashboard); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedDashboard); - importResult.setOldEntity(existingDashboard); - return importResult; + return dashboardService.saveDashboard(dashboard); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java index c72ed48b2f..c41e586b83 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.expimp.imp.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.impl.DeviceExportData; @@ -25,7 +24,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,44 +33,14 @@ public class DeviceImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceExportData exportData) { - Device device = exportData.getDevice(); - Device existingDevice = findByExternalId(tenantId, device.getId()); // FIXME: !!! - // what if exporting and importing back already exported entity ? (save version and then load it back) - /* - * export entity -> id from env1 -> import this entity -> ... - * - * maybe find not only by external id but by internal too ? but then what if we will try - * */ - - device.setExternalId(device.getId()); - device.setTenantId(tenantId); - - if (existingDevice == null) { - device.setId(null); - device.setCustomerId(null); // FIXME: find and set customer - } else { - device.setId(existingDevice.getId()); - device.setCustomerId(existingDevice.getCustomerId()); - } - - // TODO or maybe set as additional config whether to update related entities when device already exists ? - // TODO: or also whether to ignore not found internal ids - - // FIXME: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants - + protected Device prepareAndSaveEntity(TenantId tenantId, Device device, Device existingDevice, DeviceExportData exportData) { + device.setCustomerId(getInternalId(tenantId, device.getCustomerId())); device.setDeviceProfileId(getInternalId(tenantId, device.getDeviceProfileId())); device.setFirmwareId(getInternalId(tenantId, device.getFirmwareId())); device.setSoftwareId(getInternalId(tenantId, device.getSoftwareId())); - Device savedDevice = deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedDevice); - importResult.setOldEntity(existingDevice); - return importResult; + return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java index 99efefe812..8be8bf6b11 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -35,30 +34,13 @@ public class DeviceProfileImportService extends AbstractEntityImportService importEntity(TenantId tenantId, DeviceProfileExportData exportData) { - DeviceProfile deviceProfile = exportData.getDeviceProfile(); - DeviceProfile existingDeviceProfile = findByExternalId(tenantId, deviceProfile.getId()); - - deviceProfile.setExternalId(deviceProfile.getId()); - deviceProfile.setTenantId(tenantId); - - if (existingDeviceProfile == null) { - deviceProfile.setId(null); - } else { - deviceProfile.setId(existingDeviceProfile.getId()); - } - + protected DeviceProfile prepareAndSaveEntity(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfile existingDeviceProfile, DeviceProfileExportData exportData) { deviceProfile.setDefaultRuleChainId(getInternalId(tenantId, deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(getInternalId(tenantId, deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(getInternalId(tenantId, deviceProfile.getFirmwareId())); deviceProfile.setSoftwareId(getInternalId(tenantId, deviceProfile.getSoftwareId())); - DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); - - EntityImportResult importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedDeviceProfile); - importResult.setOldEntity(existingDeviceProfile); - return importResult; + return deviceProfileService.saveDeviceProfile(deviceProfile); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java index 92daa19a00..dbc6b82545 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java @@ -26,7 +26,6 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; @Service @TbCoreComponent @@ -38,27 +37,16 @@ public class RuleChainImportService extends AbstractEntityImportService importEntity(TenantId tenantId, RuleChainExportData exportData) { - RuleChain ruleChain = exportData.getRuleChain(); - RuleChain existingRuleChain = findByExternalId(tenantId, ruleChain.getId()); - - ruleChain.setExternalId(ruleChain.getId()); - ruleChain.setTenantId(tenantId); + protected RuleChain prepareAndSaveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData) { ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting - - if (existingRuleChain == null) { - ruleChain.setId(null); - } else { - ruleChain.setId(existingRuleChain.getId()); + if (existingRuleChain != null) { + ruleChainService.deleteRuleNodes(tenantId, existingRuleChain.getId()); } - RuleChain savedRuleChain = ruleChainService.saveRuleChain(ruleChain); + ruleChain = ruleChainService.saveRuleChain(ruleChain); - if (ruleChain.getId() != null) { - ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); - } RuleChainMetaData metaData = exportData.getMetaData(); - metaData.setRuleChainId(savedRuleChain.getId()); + metaData.setRuleChainId(ruleChain.getId()); metaData.getNodes().forEach(ruleNode -> { ruleNode.setId(null); ruleNode.setRuleChainId(null); @@ -70,10 +58,7 @@ public class RuleChainImportService extends AbstractEntityImportService importResult = new EntityImportResult<>(); - importResult.setSavedEntity(savedRuleChain); - importResult.setOldEntity(existingRuleChain); - return importResult; + return ruleChain; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index 065e0c5db5..f87b44b0eb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -22,12 +22,13 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -public class Customer extends ContactBased implements HasTenantId { +public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 4b69534bc0..c8b0df98b4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -19,9 +19,10 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; -public class Dashboard extends DashboardInfo { +public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 519cffc39a..3bd369591f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -25,6 +25,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -40,7 +41,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) @Slf4j -public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage { +public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 6161cd6a5c..0bd770d02d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -24,6 +24,7 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -43,7 +44,7 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn @ToString(exclude = {"image", "profileDataBytes"}) @EqualsAndHashCode(callSuper = true) @Slf4j -public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage { +public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 6998485460273302018L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index e0924c63ee..694a0162bc 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +36,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) -public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId { +public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java index cab75313d4..fdf08b2cdb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java @@ -18,12 +18,11 @@ package org.thingsboard.server.common.data.export; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; import java.util.List; import java.util.Map; @Data public class EntitiesExportResponse { - private Map>>> exportData; + private Map>>> exportData; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java index 4fd1a5c732..b428c3e6b5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) @@ -40,7 +39,7 @@ import org.thingsboard.server.common.data.id.HasId; @Type(name = "DASHBOARD", value = DashboardExportData.class), @Type(name = "CUSTOMER", value = CustomerExportData.class) }) -public interface EntityExportData> { +public interface EntityExportData> { @JsonIgnore E getMainEntity(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java new file mode 100644 index 0000000000..89b0cc3de7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.export; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.springframework.data.annotation.Transient; +import org.thingsboard.server.common.data.HasName; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ExportableEntity extends HasId, HasTenantId, HasName { + + @Transient @JsonIgnore + I getId(); + @Transient @JsonIgnore + void setId(I id); + + I getExternalId(); + void setExternalId(I externalId); + + TenantId getTenantId(); + void setTenantId(TenantId tenantId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 7ff2b0dc2b..370087deb3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; +import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Data @EqualsAndHashCode(callSuper = true) @Slf4j -public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { +public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, ExportableEntity { private static final long serialVersionUID = -5656679015121935465L; From 5ee056ff8da694659a5ceb98f3cf92209a40126a Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 25 Mar 2022 15:32:54 +0200 Subject: [PATCH 005/156] Export/import API refactoring --- .../EntitiesExportImportController.java | 70 +++++++++---------- .../DefaultEntitiesExportImportService.java | 37 ++++++---- .../expimp/EntitiesExportImportService.java | 6 +- .../expimp/ExportableEntitiesService.java | 2 + .../expimp/exp/EntityExportService.java | 6 +- .../expimp/exp/EntityExportSettings.java | 17 ++--- .../exp/impl/AbstractEntityExportService.java | 63 +++++++++++++++++ .../expimp/exp/impl/AssetExportService.java | 17 +---- .../exp/impl/CustomerExportService.java | 17 +---- .../exp/impl/DashboardExportService.java | 17 +---- .../expimp/exp/impl/DeviceExportService.java | 18 +++-- .../exp/impl/DeviceProfileExportService.java | 17 +---- .../exp/impl/RuleChainExportService.java | 16 ++--- .../expimp/imp/EntityImportService.java | 8 ++- .../expimp/imp/EntityImportSettings.java | 18 ++--- .../imp/impl/AbstractEntityImportService.java | 34 +++++---- .../expimp/imp/impl/AssetImportService.java | 3 +- .../imp/impl/CustomerImportService.java | 3 +- .../imp/impl/DashboardImportService.java | 5 +- .../expimp/imp/impl/DeviceImportService.java | 3 +- .../imp/impl/DeviceProfileImportService.java | 3 +- .../imp/impl/RuleChainImportService.java | 7 +- .../server/common/data/Customer.java | 2 +- .../common/data/export/EntityExportData.java | 13 ++-- .../common/data/export/ExportableEntity.java | 4 -- .../data/export/impl/AssetExportData.java | 11 +-- .../data/export/impl/CustomerExportData.java | 11 +-- .../data/export/impl/DashboardExportData.java | 11 +-- .../data/export/impl/DeviceExportData.java | 10 +-- .../export/impl/DeviceProfileExportData.java | 11 +-- .../data/export/impl/RuleChainExportData.java | 10 +-- .../server/common/data/id/IdBased.java | 37 +++++----- .../server/dao/ExportableEntityDao.java | 2 +- 33 files changed, 265 insertions(+), 244 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java => application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java (65%) create mode 100644 application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java rename common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java => application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java (61%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index a3f26bf7b0..96bc70656f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,11 +17,13 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; +import org.apache.commons.collections.CollectionUtils; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -29,14 +31,18 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.EntitiesExportImportService; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; import org.thingsboard.server.service.expimp.imp.EntityImportResult; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.UUID; +import java.util.stream.Collectors; @RestController @RequestMapping("/api/entities") @@ -49,66 +55,60 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") @PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid) throws ThingsboardException { + public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") + @PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportInboundRelations(exportInboundRelations) + .build(); try { - return exportEntity(getCurrentUser(), entityId); + return exportEntity(getCurrentUser(), entityId, exportSettings); } catch (Exception e) { throw handleException(e); } } -// @PostMapping("/export/batch") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntitiesExportResponse exportEntities(@RequestBody EntitiesExportRequest exportRequest) throws ThingsboardException { -// TenantId tenantId = getTenantId(); -// -// EntitiesExportResponse exportResponse = new EntitiesExportResponse(); -// -// Map>>> result = new HashMap<>(); -// exportRequest.getEntities().forEach((entityType, entityIds) -> { -// List>> exportDataForEntityType = new LinkedList<>(); -// entityIds.forEach(entityId -> { -// EntityExportData> exportData = exportImportService.exportEntity(tenantId, entityId); -// exportDataForEntityType.add(exportData); -// }); -// result.put(entityType, exportDataForEntityType); -// }); -// -// exportResponse.setExportData(result); -// return exportResponse; -// } - // TODO: export and import of batches - // TODO: api to export and import whole customer, whole tenant - - @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData) throws ThingsboardException { + public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData, + @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { + EntityImportSettings importSettings = EntityImportSettings.builder() + .importInboundRelations(importInboundRelations) + .build(); try { - return importEntity(getCurrentUser(), exportData); + return importEntity(getCurrentUser(), exportData, importSettings); } catch (Exception e) { throw handleException(e); } } - private EntityExportData> exportEntity(SecurityUser user, EntityId entityId) throws ThingsboardException { + // TODO [viacheslav]: export and import of batches + // TODO [viacheslav]: api to export and import whole customer, whole tenant + + private EntityExportData> exportEntity(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); - return exportImportService.exportEntity(getTenantId(), entityId); + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); } - private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData) throws ThingsboardException { + private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) { - checkEntityId(existingEntity.getId(), Operation.WRITE); // todo maybe need to extract permission check to BaseController and put there permission checks from other controllers + if (existingEntity != null) {// todo [viacheslav] maybe need to extract permission check to BaseController and put there permission checks from other controllers + checkEntityId(existingEntity.getId(), Operation.WRITE); + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + for (EntityId fromId : exportData.getInboundRelations().stream().map(EntityRelation::getFrom).collect(Collectors.toSet())) { + // FIXME [viacheslav]: fromId is external +// checkEntityId(fromId, Operation.WRITE); + } + } } else { checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } - EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData); + EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData, importSettings); onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); return importResult; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java index c6fe03a750..0136bd1d52 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java @@ -25,56 +25,63 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Optional; -// FIXME: review packages and classes naming +// FIXME [viacheslav]: review packages and classes naming @Service @TbCoreComponent public class DefaultEntitiesExportImportService implements EntitiesExportImportService { - private final Map> exportServices = new HashMap<>(); + private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); - // TODO: export and import of the whole tenant - // TODO: export and import of the whole customer ? - // TODO: relations export and import + // TODO [viacheslav]: export and import of the whole tenant + // TODO [viacheslav]: export and import of the whole customer ? @Override - public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId) { + public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { EntityType entityType = entityId.getEntityType(); - EntityExportService exportService = getExportService(entityType); + EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(tenantId, entityId); + return exportService.getExportData(tenantId, entityId, exportSettings); } - // FIXME: somehow validate export data + // FIXME [viacheslav]: somehow validate export data @Override - public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData) { + public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - return importService.importEntity(tenantId, exportData); + return importService.importEntity(tenantId, exportData, importSettings); } + @Override + public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { + ExportableEntityDao dao = getDao(id.getEntityType()); + return dao.findByTenantIdAndId(tenantId.getId(), id.getId()); + } + @Override public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { ExportableEntityDao dao = getDao(externalId.getEntityType()); return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) - .orElseGet(() -> dao.findByTenantIdAndId(tenantId.getId(), externalId.getId())); + .orElseGet(() -> findEntityById(tenantId, externalId)); } @SuppressWarnings("unchecked") - private > EntityExportService getExportService(EntityType entityType) { - return (EntityExportService) exportServices.get(entityType); + private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") @@ -88,7 +95,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } @Autowired - private void setServices(Collection> exportServices, + private void setServices(Collection> exportServices, Collection> importServices, Collection> daos) { exportServices.forEach(entityExportService -> { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java index 10361865a2..752d4b6315 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java @@ -19,12 +19,14 @@ import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; import org.thingsboard.server.service.expimp.imp.EntityImportResult; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; public interface EntitiesExportImportService extends ExportableEntitiesService { - , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId); + , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); - , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData); + , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java index ca401db64b..17151e3889 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.id.TenantId; public interface ExportableEntitiesService { + , I extends EntityId> E findEntityById(TenantId tenantId, I id); + , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java index bdd525d5d9..7c7fc49240 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java @@ -21,11 +21,9 @@ import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -public interface EntityExportService> { +public interface EntityExportService, D extends EntityExportData> { - // FIXME: export relations - // FIXME: get rid of boilerplate - EntityExportData getExportData(TenantId tenantId, I entityId); + D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings); EntityType getEntityType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java similarity index 65% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java rename to application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java index fdf08b2cdb..c70b0b2216 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportResponse.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.service.expimp.exp; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; -import java.util.Map; +import lombok.NoArgsConstructor; @Data -public class EntitiesExportResponse { - private Map>>> exportData; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityExportSettings { + private boolean exportInboundRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java new file mode 100644 index 0000000000..f6c2f5d5f7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.expimp.exp.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.export.EntityExportData; +import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.service.expimp.ExportableEntitiesService; +import org.thingsboard.server.service.expimp.exp.EntityExportService; +import org.thingsboard.server.service.expimp.exp.EntityExportSettings; + +import java.util.List; + +public abstract class AbstractEntityExportService, D extends EntityExportData> implements EntityExportService { + + @Autowired @Lazy + private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + + + @Override + public final D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { + D exportData = newExportData(); + + E mainEntity = exportableEntitiesService.findEntityById(tenantId, entityId); + exportData.setMainEntity(mainEntity); + setRelatedEntities(tenantId, mainEntity, exportData); + if (exportSettings.isExportInboundRelations()) { + exportData.setInboundRelations(getInboundRelations(tenantId, entityId)); + } + + return exportData; + } + + protected List getInboundRelations(TenantId tenantId, I entityId) { + return relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); // FIXME [viacheslav]: RelationTypeGroup + } + + protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {} + + protected abstract D newExportData(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java index bb4b68bb29..ec1b2dddc3 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class AssetExportService implements EntityExportService { - - private final AssetService assetService; - +public class AssetExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, AssetId assetId) { - AssetExportData exportData = new AssetExportData(); - exportData.setAsset(assetService.findAssetById(tenantId, assetId)); - return exportData; + protected AssetExportData newExportData() { + return new AssetExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java index 7430f9ddd3..32e33f757c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.CustomerExportData; import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class CustomerExportService implements EntityExportService { - - private final CustomerService customerService; - +public class CustomerExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, CustomerId customerId) { - CustomerExportData exportData = new CustomerExportData(); - exportData.setCustomer(customerService.findCustomerById(tenantId, customerId)); - return exportData; + protected CustomerExportData newExportData() { + return new CustomerExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java index 87959429d9..be86c41c9a 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.DashboardExportData; import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class DashboardExportService implements EntityExportService { - - private final DashboardService dashboardService; - +public class DashboardExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, DashboardId dashboardId) { - DashboardExportData exportData = new DashboardExportData(); - exportData.setDashboard(dashboardService.findDashboardById(tenantId, dashboardId)); - return exportData; + protected DashboardExportData newExportData() { + return new DashboardExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java index 860aaa9b1d..c3542dc95b 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java @@ -19,30 +19,28 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; -import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceExportService implements EntityExportService { +public class DeviceExportService extends AbstractEntityExportService { - private final DeviceService deviceService; private final DeviceCredentialsService deviceCredentialsService; @Override - public EntityExportData getExportData(TenantId tenantId, DeviceId deviceId) { - DeviceExportData exportData = new DeviceExportData(); - exportData.setDevice(deviceService.findDeviceById(tenantId, deviceId)); - exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId)); - return exportData; + protected void setRelatedEntities(TenantId tenantId, Device device, DeviceExportData exportData) { + exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, device.getId())); + } + + @Override + protected DeviceExportData newExportData() { + return new DeviceExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java index 425dbfd5be..03787ab30e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java @@ -15,31 +15,20 @@ */ package org.thingsboard.server.service.expimp.exp.impl; -import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent -@RequiredArgsConstructor -public class DeviceProfileExportService implements EntityExportService { - - private final DeviceProfileService deviceProfileService; - +public class DeviceProfileExportService extends AbstractEntityExportService { @Override - public EntityExportData getExportData(TenantId tenantId, DeviceProfileId deviceProfileId) { - DeviceProfileExportData exportData = new DeviceProfileExportData(); - exportData.setDeviceProfile(deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId)); - return exportData; + protected DeviceProfileExportData newExportData() { + return new DeviceProfileExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java index 7a85281d63..0e2d24e990 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java @@ -18,29 +18,29 @@ package org.thingsboard.server.service.expimp.exp.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; @Service @TbCoreComponent @RequiredArgsConstructor -public class RuleChainExportService implements EntityExportService { +public class RuleChainExportService extends AbstractEntityExportService { private final RuleChainService ruleChainService; @Override - public EntityExportData getExportData(TenantId tenantId, RuleChainId ruleChainId) { - RuleChainExportData exportData = new RuleChainExportData(); - exportData.setRuleChain(ruleChainService.findRuleChainById(tenantId, ruleChainId)); - exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId)); - return exportData; + protected void setRelatedEntities(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData) { + exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChain.getId())); + } + + @Override + protected RuleChainExportData newExportData() { + return new RuleChainExportData(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java index 96a0f732ed..693ff310bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java @@ -23,7 +23,13 @@ import org.thingsboard.server.common.data.id.TenantId; public interface EntityImportService, D extends EntityExportData> { - EntityImportResult importEntity(TenantId tenantId, D exportData); + /* + * TODO [viacheslav]: should be as options: + * to update related entities e.g. firmware or device profile if entity already exists + * to delete current relations when importing new + * to ignore when cannot find linked entity by external id + * */ + EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings); EntityType getEntityType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java similarity index 61% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java index d3502695c0..1b29139e6b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntitiesExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.service.expimp.imp; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; -import java.util.Map; +import lombok.NoArgsConstructor; @Data -public class EntitiesExportRequest { - // todo: should be processed in a specific order, e.g. device profiles, then devices, then relations, etc. - private Map> entities; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityImportSettings { + private boolean importInboundRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java index fd635e239c..fd27feeea9 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.expimp.imp.impl; +import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.export.EntityExportData; @@ -22,23 +23,22 @@ import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.expimp.ExportableEntitiesService; import org.thingsboard.server.service.expimp.imp.EntityImportResult; import org.thingsboard.server.service.expimp.imp.EntityImportService; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + - // FIXME what if exporting and importing back already exported entity ? (save version and then load it back) - /* - * export entity -> id from env1 -> import this entity -> ... - * - * maybe find not only by external id but by internal too ? but then what if we will try - * */ @Override - public final EntityImportResult importEntity(TenantId tenantId, D exportData) { + public final EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { E entity = exportData.getMainEntity(); E existingEntity = findByExternalId(tenantId, entity.getId()); @@ -51,7 +51,18 @@ public abstract class AbstractEntityImportService { + relation.setTo(savedEntity.getId()); + relation.setFrom(getInternalId(tenantId, relation.getFrom())); + relationService.saveRelation(tenantId, relation); + }); + } EntityImportResult importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -59,17 +70,14 @@ public abstract class AbstractEntityImportService ID getInternalId(TenantId tenantId, ID externalId) { if (externalId == null) { return null; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java index 4e6fa2b33e..f228b9be5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.expimp.imp.EntityImportSettings; @Service @TbCoreComponent @@ -34,7 +35,7 @@ public class AssetImportService extends AbstractEntityImportService { // ruleChainConnectionInfo.setTargetRuleChainId(); - // TODO: check if this thing is needed for "Other Rule Chain Node" - // TODO: and check import of tenant rule chains + // TODO [viacheslav]: check if this thing is needed for "Other Rule Chain Node" + // TODO [viacheslav]: and check import of tenant rule chains }); ruleChainService.saveRuleChainMetaData(tenantId, metaData); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index f87b44b0eb..c2f23a030f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -40,7 +40,7 @@ public class Customer extends ContactBased implements HasTenantId, E private TenantId tenantId; @Getter @Setter - private CustomerId externalId; // FIXME: add to hashcode, equals, etc + private CustomerId externalId; // FIXME [viacheslav]: add to hashcode, equals, etc public Customer() { super(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java index b428c3e6b5..261178d913 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.export.impl.CustomerExportData; @@ -28,6 +29,9 @@ import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; + +import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) @@ -39,12 +43,13 @@ import org.thingsboard.server.common.data.id.EntityId; @Type(name = "DASHBOARD", value = DashboardExportData.class), @Type(name = "CUSTOMER", value = CustomerExportData.class) }) -public interface EntityExportData> { +@Data +public abstract class EntityExportData> { - @JsonIgnore - E getMainEntity(); + private E mainEntity; + private List inboundRelations; @JsonIgnore - EntityType getEntityType(); // fixme: maybe remove if not needed + public abstract EntityType getEntityType(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java index 89b0cc3de7..d990652298 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.export; -import com.fasterxml.jackson.annotation.JsonIgnore; -import org.springframework.data.annotation.Transient; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; @@ -25,9 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; public interface ExportableEntity extends HasId, HasTenantId, HasName { - @Transient @JsonIgnore I getId(); - @Transient @JsonIgnore void setId(I id); I getExternalId(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java index abb268eeb6..6277a56afe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class AssetExportData implements EntityExportData { - - private Asset asset; - - @Override - public Asset getMainEntity() { - return asset; - } +public class AssetExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java index 02ec9e7efa..3de1dbc1b9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class CustomerExportData implements EntityExportData { - - private Customer customer; - - @Override - public Customer getMainEntity() { - return customer; - } +public class CustomerExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java index f75f2479f6..7310ad1f92 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class DashboardExportData implements EntityExportData { - - private Dashboard dashboard; - - @Override - public Dashboard getMainEntity() { - return dashboard; - } +public class DashboardExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java index ff3741dabc..7d4d404ebe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java @@ -16,22 +16,18 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.security.DeviceCredentials; +@EqualsAndHashCode(callSuper = true) @Data -public class DeviceExportData implements EntityExportData { +public class DeviceExportData extends EntityExportData { - private Device device; private DeviceCredentials credentials; - @Override - public Device getMainEntity() { - return device; - } - @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java index a5d7872cc6..4ebae37b8e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java @@ -16,19 +16,14 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; +@EqualsAndHashCode(callSuper = true) @Data -public class DeviceProfileExportData implements EntityExportData { - - private DeviceProfile deviceProfile; - - @Override - public DeviceProfile getMainEntity() { - return deviceProfile; - } +public class DeviceProfileExportData extends EntityExportData { @Override public EntityType getEntityType() { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java index 101f516881..d52c68b99e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java @@ -16,22 +16,18 @@ package org.thingsboard.server.common.data.export.impl; import lombok.Data; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +@EqualsAndHashCode(callSuper = true) @Data -public class RuleChainExportData implements EntityExportData { +public class RuleChainExportData extends EntityExportData { - private RuleChain ruleChain; private RuleChainMetaData metaData; - @Override - public RuleChain getMainEntity() { - return ruleChain; - } - @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java index cfe6918a7d..5c99ad8ddb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java @@ -16,33 +16,34 @@ package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; -import java.io.Serializable; import java.util.UUID; public abstract class IdBased implements HasId { protected I id; - - public IdBased() { - super(); - } - - public IdBased(I id) { - super(); - this.id = id; - } - public void setId(I id) { - this.id = id; - } + public IdBased() { + super(); + } - public I getId() { - return id; - } + public IdBased(I id) { + super(); + this.id = id; + } + + @JsonSetter + public void setId(I id) { + this.id = id; + } + + public I getId() { + return id; + } - @JsonIgnore - public UUID getUuidId() { + @JsonIgnore + public UUID getUuidId() { if (id != null) { return id.getId(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index c7741ec013..4e0822936b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -24,7 +24,7 @@ public interface ExportableEntityDao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); T findByTenantIdAndId(UUID tenantId, UUID id); - // fixme: get rid of boilerplate ? + // fixme [viacheslav]: get rid of boilerplate ? EntityType getEntityType(); From 040b6a31adb7bae03139b843697b2ae4e5cb0849 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 25 Mar 2022 15:54:07 +0200 Subject: [PATCH 006/156] Rename packages for export/import --- .../EntitiesExportImportController.java | 12 ++++++------ .../DefaultEntitiesExportImportService.java | 16 ++++++++-------- .../EntitiesExportImportService.java | 12 ++++++------ .../ExportableEntitiesService.java | 4 ++-- .../exporting}/EntityExportService.java | 6 +++--- .../exporting}/EntityExportSettings.java | 2 +- .../exporting/data}/AssetExportData.java | 3 +-- .../exporting/data}/CustomerExportData.java | 3 +-- .../exporting/data}/DashboardExportData.java | 3 +-- .../exporting/data}/DeviceExportData.java | 3 +-- .../exporting/data}/DeviceProfileExportData.java | 3 +-- .../exporting/data}/EntityExportData.java | 9 ++------- .../exporting/data}/RuleChainExportData.java | 3 +-- .../impl/AbstractEntityExportService.java | 12 ++++++------ .../exporting}/impl/AssetExportService.java | 4 ++-- .../exporting}/impl/CustomerExportService.java | 4 ++-- .../exporting}/impl/DashboardExportService.java | 4 ++-- .../exporting}/impl/DeviceExportService.java | 4 ++-- .../impl/DeviceProfileExportService.java | 4 ++-- .../exporting}/impl/RuleChainExportService.java | 4 ++-- .../importing}/EntityImportResult.java | 4 ++-- .../importing}/EntityImportService.java | 6 +++--- .../importing}/EntityImportSettings.java | 2 +- .../impl/AbstractEntityImportService.java | 14 +++++++------- .../importing}/impl/AssetImportService.java | 6 +++--- .../importing}/impl/CustomerImportService.java | 6 +++--- .../importing}/impl/DashboardImportService.java | 6 +++--- .../importing}/impl/DeviceImportService.java | 6 +++--- .../impl/DeviceProfileImportService.java | 6 +++--- .../importing}/impl/RuleChainImportService.java | 6 +++--- .../thingsboard/server/common/data/Customer.java | 1 - .../server/common/data/Dashboard.java | 1 - .../thingsboard/server/common/data/Device.java | 1 - .../server/common/data/DeviceProfile.java | 1 - .../data/{export => }/ExportableEntity.java | 4 +--- .../server/common/data/asset/Asset.java | 2 +- .../server/common/data/rule/RuleChain.java | 2 +- 37 files changed, 86 insertions(+), 103 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{expimp => exportimport}/DefaultEntitiesExportImportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp => exportimport}/EntitiesExportImportService.java (72%) rename application/src/main/java/org/thingsboard/server/service/{expimp => exportimport}/ExportableEntitiesService.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/EntityExportService.java (83%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/EntityExportSettings.java (93%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/AssetExportData.java (88%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/CustomerExportData.java (88%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/DashboardExportData.java (88%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/DeviceExportData.java (89%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/DeviceProfileExportData.java (89%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/EntityExportData.java (79%) rename {common/data/src/main/java/org/thingsboard/server/common/data/export/impl => application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data}/RuleChainExportData.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/AbstractEntityExportService.java (83%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/AssetExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/CustomerExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/DashboardExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/DeviceProfileExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/exp => exportimport/exporting}/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/EntityImportResult.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/EntityImportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/EntityImportSettings.java (93%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/AbstractEntityImportService.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/AssetImportService.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/CustomerImportService.java (87%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/DashboardImportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/DeviceImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/DeviceProfileImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/{expimp/imp => exportimport/importing}/impl/RuleChainImportService.java (91%) rename common/data/src/main/java/org/thingsboard/server/common/data/{export => }/ExportableEntity.java (86%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 96bc70656f..fc3b0b712b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -26,17 +26,17 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.EntitiesExportImportService; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 0136bd1d52..4e9c42cffd 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -13,22 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp; +package org.thingsboard.server.service.exportimport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.exp.EntityExportService; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportService; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import java.util.Collection; import java.util.HashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index 752d4b6315..1eb085d219 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp; +package org.thingsboard.server.service.exportimport; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; public interface EntitiesExportImportService extends ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java index 17151e3889..bb8f30c4e1 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp; +package org.thingsboard.server.service.exportimport; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java index 7c7fc49240..929092e90e 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp; +package org.thingsboard.server.service.exportimport.exporting; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java index c70b0b2216..3755ce801d 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp; +package org.thingsboard.server.service.exportimport.exporting; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java index 6277a56afe..4c4872760b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/AssetExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java index 3de1dbc1b9..1fc820814e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/CustomerExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java index 7310ad1f92..6ef45fc531 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DashboardExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java index 7d4d404ebe..5f5d8e8f72 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java index 4ebae37b8e..def6b7edbe 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/DeviceProfileExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; @EqualsAndHashCode(callSuper = true) @Data diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java similarity index 79% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index 261178d913..74116276cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.service.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -22,12 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.AssetExportData; -import org.thingsboard.server.common.data.export.impl.CustomerExportData; -import org.thingsboard.server.common.data.export.impl.DashboardExportData; -import org.thingsboard.server.common.data.export.impl.DeviceExportData; -import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; -import org.thingsboard.server.common.data.export.impl.RuleChainExportData; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java index d52c68b99e..f92616fead 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/impl/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java @@ -13,12 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export.impl; +package org.thingsboard.server.service.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java index f6c2f5d5f7..7b298a7d3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AbstractEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.expimp.ExportableEntitiesService; -import org.thingsboard.server.service.expimp.exp.EntityExportService; -import org.thingsboard.server.service.expimp.exp.EntityExportSettings; +import org.thingsboard.server.service.exportimport.ExportableEntitiesService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java index ec1b2dddc3..a48167053b 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java index 32e33f757c..518a830a21 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.CustomerExportData; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java index be86c41c9a..167b57be75 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DashboardExportData; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java index c3542dc95b..8a8bc26820 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java index 03787ab30e..ce96e3b1fe 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java index 0e2d24e990..8c173fe2ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/exp/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java @@ -13,17 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.exp.impl; +package org.thingsboard.server.service.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java index 90f5d42bc9..cc44670757 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp; +package org.thingsboard.server.service.exportimport.importing; import lombok.Data; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @Data diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java index 693ff310bf..b001290cef 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp; +package org.thingsboard.server.service.exportimport.importing; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java index 1b29139e6b..d153d1378a 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp; +package org.thingsboard.server.service.exportimport.importing; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java index fd27feeea9..075991104c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; -import org.thingsboard.server.common.data.export.EntityExportData; -import org.thingsboard.server.common.data.export.ExportableEntity; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.expimp.ExportableEntitiesService; -import org.thingsboard.server.service.expimp.imp.EntityImportResult; -import org.thingsboard.server.service.expimp.imp.EntityImportService; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.ExportableEntitiesService; +import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportResult; +import org.thingsboard.server.service.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java index f228b9be5e..6699219388 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.export.impl.AssetExportData; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java index 7a8bfa7c9d..251dd4acbf 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.CustomerExportData; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java index dcfba5f17b..b429ba7111 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DashboardExportData; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java index 50db77fb2e..01312dfa44 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceExportData; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java index 8fe7586a25..ef72d982ee 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.DeviceProfileExportData; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java index 5dfe6a8692..5fc1c8fb8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/expimp/imp/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.expimp.imp.impl; +package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.export.impl.RuleChainExportData; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.expimp.imp.EntityImportSettings; +import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index c2f23a030f..b7287c55af 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index c8b0df98b4..dce910c4ff 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; public class Dashboard extends DashboardInfo implements ExportableEntity { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 3bd369591f..678c68c115 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -25,7 +25,6 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index 0bd770d02d..891367fac4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -24,7 +24,6 @@ import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java similarity index 86% rename from common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java rename to common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index d990652298..f1e8c3754e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/export/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -13,10 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.export; +package org.thingsboard.server.common.data; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index 694a0162bc..da6d9615a1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -21,11 +21,11 @@ import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 370087deb3..74186b49bf 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -22,10 +22,10 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; -import org.thingsboard.server.common.data.export.ExportableEntity; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; From ab5d6f3c0dcb6ad301fbd26005854632c104bdc0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 30 Mar 2022 13:31:43 +0300 Subject: [PATCH 007/156] Entities export/import API refactoring --- .../EntitiesExportImportController.java | 144 ++++++++++++++---- .../DefaultEntitiesExportImportService.java | 26 ++++ .../EntitiesExportImportService.java | 4 + .../exporting/data/EntityExportData.java | 2 + .../server/common/data/Customer.java | 35 +---- .../server/common/data/Dashboard.java | 27 +--- .../server/dao/ExportableEntityDao.java | 7 +- 7 files changed, 153 insertions(+), 92 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index fc3b0b712b..ec2d4273cc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -27,23 +27,37 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.EntitiesExportImportService; import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + @RestController @RequestMapping("/api/entities") @TbCoreComponent @@ -51,7 +65,58 @@ import java.util.stream.Collectors; public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; + private final EntityQueryService entityQueryService; + + + // TODO [viacheslav]: export and import of batches + // TODO [viacheslav]: api to export and import whole customer, whole tenant + + + @PostMapping("/exportByFilter") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, + @RequestParam(defaultValue = "false") boolean exportInboundRelations, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME), EntityDataSortOrder.Direction.DESC)); + EntityDataQuery entityDataQuery = new EntityDataQuery(filter, pageLink, List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME)), + Collections.emptyList(), Collections.emptyList()); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { + // FIXME [viacheslav]: check read permission for relation fromId + return entityQueryService.findEntityDataByQuery(user, entityDataQuery).getData().stream() + .map(EntityData::getEntityId) + .map(entityId -> { + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + }) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); + } + } + @PostMapping("/exportByQuery") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List> exportEntitiesByQuery(@RequestBody EntityDataQuery query, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); +// FIXME [viacheslav]: check read permission for relation fromId + try { + return entityQueryService.findEntityDataByQuery(user, query).getData().stream() + .map(EntityData::getEntityId) + .map(entityId -> exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings)) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); + } + } @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -60,11 +125,13 @@ public class EntitiesExportImportController extends BaseController { @PathVariable("entityId") UUID entityUuid, @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(exportInboundRelations) - .build(); - try { - return exportEntity(getCurrentUser(), entityId, exportSettings); + checkEntityId(entityId, Operation.READ); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { // FIXME [viacheslav]: check read permission for relation fromId + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); } catch (Exception e) { throw handleException(e); } @@ -73,45 +140,64 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData, - @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { - EntityImportSettings importSettings = EntityImportSettings.builder() - .importInboundRelations(importInboundRelations) - .build(); + public List>> importEntity(@RequestBody List>> exportDataList, + @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityImportSettings importSettings = toImportSettings(importInboundRelations); + + for (EntityExportData> exportData : exportDataList) { + checkPermissionsForImport(user, exportData, importSettings); + } + try { - return importEntity(getCurrentUser(), exportData, importSettings); + return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); } catch (Exception e) { throw handleException(e); } } - - // TODO [viacheslav]: export and import of batches - // TODO [viacheslav]: api to export and import whole customer, whole tenant - - private EntityExportData> exportEntity(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { + public void checkPermissionsForExport(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + if (exportSettings.isExportInboundRelations()) { + for (EntityRelation entityRelation : relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON)) { + EntityId fromId = entityRelation.getFrom(); + checkEntityId(fromId, Operation.READ); + } + } } - private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { - ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) {// todo [viacheslav] maybe need to extract permission check to BaseController and put there permission checks from other controllers + public void checkPermissionsForImport(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { + ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { checkEntityId(existingEntity.getId(), Operation.WRITE); - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - for (EntityId fromId : exportData.getInboundRelations().stream().map(EntityRelation::getFrom).collect(Collectors.toSet())) { - // FIXME [viacheslav]: fromId is external -// checkEntityId(fromId, Operation.WRITE); + } else { + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + } + + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + for (EntityRelation entityRelation : exportData.getInboundRelations()) { + ExportableEntity entityFrom = exportImportService.findEntityByExternalId(user.getTenantId(), entityRelation.getFrom()); + if (entityFrom != null) { + accessControlService.checkPermission(user, Resource.of(entityFrom.getId().getEntityType()), Operation.WRITE, entityFrom.getId(), entityFrom); + } else { + throw new ThingsboardException("Relation with non-existing entity", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } } - } else { - checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } + } + - EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData, importSettings); - onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); + private EntityImportSettings toImportSettings(boolean importInboundRelations) { + EntityImportSettings importSettings = EntityImportSettings.builder() + .importInboundRelations(importInboundRelations) + .build(); + return importSettings; + } - return importResult; + private EntityExportSettings toExportSettings(boolean exportInboundRelations) { + return EntityExportSettings.builder() + .exportInboundRelations(exportInboundRelations) + .build(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 4e9c42cffd..93fa370c94 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.exportimport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @@ -31,9 +32,12 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportService import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; // FIXME [viacheslav]: review packages and classes naming @Service @@ -44,6 +48,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); + protected static final List SUPPORTED_ENTITY_TYPES = List.of( + EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, + EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD + ); + // TODO [viacheslav]: export and import of the whole tenant // TODO [viacheslav]: export and import of the whole customer ? @@ -55,7 +64,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return exportService.getExportData(tenantId, entityId, exportSettings); } + // FIXME [viacheslav]: somehow validate export data + @Transactional @Override public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { EntityType entityType = exportData.getEntityType(); @@ -64,6 +75,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importService.importEntity(tenantId, exportData, importSettings); } + @Transactional + @Override + public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { + return exportDataList.stream() + .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) + .map(exportData -> importEntity(tenantId, exportData, importSettings)) + .collect(Collectors.toList()); + } + @Override public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { @@ -81,11 +101,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); + } return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); + } return (EntityImportService) importServices.get(entityType); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index 1eb085d219..e8d4a40efe 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -23,10 +23,14 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.List; + public interface EntitiesExportImportService extends ExportableEntitiesService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); + , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings); + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index 74116276cb..f77a5fbb29 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -42,6 +43,7 @@ import java.util.List; public abstract class EntityExportData> { private E mainEntity; + @JsonInclude(JsonInclude.Include.NON_NULL) private List inboundRelations; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index b7287c55af..92e5e9ae78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +@EqualsAndHashCode(callSuper = false) public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; @@ -39,7 +41,7 @@ public class Customer extends ContactBased implements HasTenantId, E private TenantId tenantId; @Getter @Setter - private CustomerId externalId; // FIXME [viacheslav]: add to hashcode, equals, etc + private CustomerId externalId; public Customer() { super(); @@ -167,37 +169,6 @@ public class Customer extends ContactBased implements HasTenantId, E return getTitle(); } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((tenantId == null) ? 0 : tenantId.hashCode()); - result = prime * result + ((title == null) ? 0 : title.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Customer other = (Customer) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; - } - @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index dce910c4ff..59c122f3d0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,10 +17,12 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; +@EqualsAndHashCode(callSuper = false) public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; @@ -60,31 +62,6 @@ public class Dashboard extends DashboardInfo implements ExportableEntity { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); T findByTenantIdAndId(UUID tenantId, UUID id); - // fixme [viacheslav]: get rid of boilerplate ? - EntityType getEntityType(); - /* - * default > ExportableEntityRepository getExportableEntityRepository() { - * ((ExportableEntityRepository) getJpaRepository).find... - * */ + EntityType getEntityType(); } From 6dc962882e954e00cec3caf8679af88461e29d94 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 30 Mar 2022 16:42:33 +0300 Subject: [PATCH 008/156] Add more options for entity export and import settings; refactoring --- .../upgrade/{3.4 => 3.3.4}/schema_update.sql | 0 .../server/controller/BaseController.java | 2 +- .../EntitiesExportImportController.java | 49 ++--- .../exporting/EntityExportSettings.java | 1 + .../exporting/data/EntityExportData.java | 3 +- .../impl/AbstractEntityExportService.java | 12 +- .../importing/EntityImportService.java | 6 - .../importing/EntityImportSettings.java | 4 + .../impl/AbstractEntityImportService.java | 83 ++++++-- .../importing/impl/AssetImportService.java | 9 +- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 17 +- .../importing/impl/DeviceImportService.java | 15 +- .../impl/DeviceProfileImportService.java | 15 +- .../impl/RuleChainImportService.java | 8 +- .../main/resources/sql/schema-entities.sql | 194 +++++++++--------- 16 files changed, 234 insertions(+), 188 deletions(-) rename application/src/main/data/upgrade/{3.4 => 3.3.4}/schema_update.sql (100%) diff --git a/application/src/main/data/upgrade/3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql similarity index 100% rename from application/src/main/data/upgrade/3.4/schema_update.sql rename to application/src/main/data/upgrade/3.3.4/schema_update.sql diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index e244bcf680..7416444104 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -924,7 +924,7 @@ public abstract class BaseController { } } - public & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index ec2d4273cc..f7513eb2d5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -72,9 +72,28 @@ public class EntitiesExportImportController extends BaseController { // TODO [viacheslav]: api to export and import whole customer, whole tenant + @PostMapping("/export/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") + @PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + checkEntityId(entityId, Operation.READ); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { // FIXME [viacheslav]: check read permission for relation fromId + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + } catch (Exception e) { + throw handleException(e); + } + } + @PostMapping("/exportByFilter") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, + public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, // TODO [viacheslav]: exportInboundRelations, exportOutboundRelations @RequestParam(defaultValue = "false") boolean exportInboundRelations, @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { @@ -118,25 +137,6 @@ public class EntitiesExportImportController extends BaseController { } } - @PostMapping("/export/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") - @PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - checkEntityId(entityId, Operation.READ); - - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); - - try { // FIXME [viacheslav]: check read permission for relation fromId - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); - } catch (Exception e) { - throw handleException(e); - } - } - @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -150,7 +150,11 @@ public class EntitiesExportImportController extends BaseController { } try { - return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); + List>> importResultList = exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); + importResultList.forEach(entityImportResult -> { + onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); + }); + return importResultList; } catch (Exception e) { throw handleException(e); } @@ -188,10 +192,9 @@ public class EntitiesExportImportController extends BaseController { private EntityImportSettings toImportSettings(boolean importInboundRelations) { - EntityImportSettings importSettings = EntityImportSettings.builder() + return EntityImportSettings.builder() .importInboundRelations(importInboundRelations) .build(); - return importSettings; } private EntityExportSettings toExportSettings(boolean exportInboundRelations) { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java index 3755ce801d..f4eea9913a 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java @@ -26,4 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityExportSettings { private boolean exportInboundRelations; + private boolean exportOutboundRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index f77a5fbb29..b9cc8c71a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -39,12 +39,13 @@ import java.util.List; @Type(name = "DASHBOARD", value = DashboardExportData.class), @Type(name = "CUSTOMER", value = CustomerExportData.class) }) +@JsonInclude(JsonInclude.Include.NON_NULL) @Data public abstract class EntityExportData> { private E mainEntity; - @JsonInclude(JsonInclude.Include.NON_NULL) private List inboundRelations; + private List outboundRelations; @JsonIgnore public abstract EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java index 7b298a7d3c..ee6437c7dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java @@ -45,17 +45,19 @@ public abstract class AbstractEntityExportService inboundRelations = relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); + exportData.setInboundRelations(inboundRelations); + } + if (exportSettings.isExportOutboundRelations()) { + List outboundRelations = relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON); + exportData.setOutboundRelations(outboundRelations); } return exportData; } - protected List getInboundRelations(TenantId tenantId, I entityId) { - return relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); // FIXME [viacheslav]: RelationTypeGroup - } - protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {} protected abstract D newExportData(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java index b001290cef..8697b04f59 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java @@ -23,12 +23,6 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa public interface EntityImportService, D extends EntityExportData> { - /* - * TODO [viacheslav]: should be as options: - * to update related entities e.g. firmware or device profile if entity already exists - * to delete current relations when importing new - * to ignore when cannot find linked entity by external id - * */ EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings); EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java index d153d1378a..22b67bcbb4 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java @@ -26,4 +26,8 @@ import lombok.NoArgsConstructor; @Builder public class EntityImportSettings { private boolean importInboundRelations; + private boolean importOutboundRelations; + private boolean removeExistingRelationsAndSaveNew; + + private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java index 075991104c..b521074bab 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java @@ -18,10 +18,13 @@ package org.thingsboard.server.service.exportimport.importing.impl; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.exportimport.ExportableEntitiesService; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; @@ -29,6 +32,11 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportService; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy @@ -36,11 +44,11 @@ public abstract class AbstractEntityImportService importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { + public EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { E entity = exportData.getMainEntity(); - E existingEntity = findByExternalId(tenantId, entity.getId()); + E existingEntity = exportableEntitiesService.findEntityByExternalId(tenantId, entity.getId()); entity.setExternalId(entity.getId()); entity.setTenantId(tenantId); @@ -51,18 +59,19 @@ public abstract class AbstractEntityImportService() { + @Override + public ID get(TenantId tenantId, Function idExtractor) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { + return getInternalId(tenantId, idExtractor.apply(entity)); + } else { + return idExtractor.apply(existingEntity); + } } - exportData.getInboundRelations().forEach(relation -> { - relation.setTo(savedEntity.getId()); - relation.setFrom(getInternalId(tenantId, relation.getFrom())); - relationService.saveRelation(tenantId, relation); - }); - } + }); + + E savedEntity = saveEntity(tenantId, entity, existingEntity, exportData); + importRelations(tenantId, savedEntity, existingEntity, exportData, importSettings); EntityImportResult importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -70,16 +79,46 @@ public abstract class AbstractEntityImportService idProvider) {} + + protected abstract E saveEntity(TenantId tenantId, E entity, E existingEntity, D exportData); + + private void importRelations(TenantId tenantId, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) { + List newRelations = new LinkedList<>(); - private E findByExternalId(TenantId tenantId, I externalId) { - return exportableEntitiesService.findEntityByExternalId(tenantId, externalId); + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + newRelations.addAll(exportData.getInboundRelations().stream() + .peek(relation -> { + relation.setTo(savedEntity.getId()); + relation.setFrom(getInternalId(tenantId, relation.getFrom())); + }) + .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelationsAndSaveNew() && existingEntity != null) { + relationService.findByTo(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { + relationService.deleteRelation(tenantId, existingRelation); + }); + } + } + if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { + newRelations.addAll(exportData.getOutboundRelations().stream() + .peek(relation -> { + relation.setTo(getInternalId(tenantId, relation.getTo())); + relation.setFrom(savedEntity.getId()); + }) + .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelationsAndSaveNew() && existingEntity != null) { + relationService.findByFrom(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { + relationService.deleteRelation(tenantId, existingRelation); + }); + } + } + + newRelations.forEach(relation -> relationService.saveRelation(tenantId, relation)); } - // FIXME [viacheslav]: review use cases for version controlling: in the same tenant, between tenants, between environments and different tenants - protected final ID getInternalId(TenantId tenantId, ID externalId) { - if (externalId == null) { + private ID getInternalId(TenantId tenantId, ID externalId) { + if (externalId == null || externalId.isNullUid()) { return null; } HasId entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); @@ -89,4 +128,8 @@ public abstract class AbstractEntityImportService { + I get(TenantId tenantId, Function idExtractor); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java index 6699219388..0d6e6d01b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,11 +32,13 @@ public class AssetImportService extends AbstractEntityImportService idProvider) { + asset.setCustomerId(idProvider.get(tenantId, Asset::getCustomerId)); + } + @Override + protected Asset saveEntity(TenantId tenantId, Asset asset, Asset existingAsset, AssetExportData exportData) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java index 251dd4acbf..22cdc10204 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,9 +32,8 @@ public class CustomerImportService extends AbstractEntityImportService idProvider) { +// if (existingDashboard == null) { +// dashboard.setAssignedCustomers(null); // FIXME [viacheslav]: need to assign dashboard to customers ? +// } else { +// dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); +// } + } + @Override + protected Dashboard saveEntity(TenantId tenantId, Dashboard dashboard, Dashboard existingDashboard, DashboardExportData exportData) { return dashboardService.saveDashboard(dashboard); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java index 01312dfa44..8c02edec05 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,14 +32,16 @@ public class DeviceImportService extends AbstractEntityImportService idProvider) { + device.setCustomerId(idProvider.get(tenantId, Device::getCustomerId)); + device.setDeviceProfileId(idProvider.get(tenantId, Device::getDeviceProfileId)); + device.setFirmwareId(idProvider.get(tenantId, Device::getFirmwareId)); + device.setSoftwareId(idProvider.get(tenantId, Device::getSoftwareId)); + } + @Override + protected Device saveEntity(TenantId tenantId, Device device, Device existingDevice, DeviceExportData exportData) { return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java index ef72d982ee..9981607481 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -33,14 +32,16 @@ public class DeviceProfileImportService extends AbstractEntityImportService idProvider) { + deviceProfile.setDefaultRuleChainId(idProvider.get(tenantId, DeviceProfile::getDefaultRuleChainId)); + deviceProfile.setDefaultDashboardId(idProvider.get(tenantId, DeviceProfile::getDefaultDashboardId)); + deviceProfile.setFirmwareId(idProvider.get(tenantId, DeviceProfile::getFirmwareId)); + deviceProfile.setSoftwareId(idProvider.get(tenantId, DeviceProfile::getSoftwareId)); + } + @Override + protected DeviceProfile saveEntity(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfile existingDeviceProfile, DeviceProfileExportData exportData) { return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java index 5fc1c8fb8c..6be2fc4317 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; @@ -26,7 +25,6 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; @Service @TbCoreComponent @@ -35,10 +33,12 @@ public class RuleChainImportService extends AbstractEntityImportService idProvider) { + } - @Transactional @Override - protected RuleChain prepareAndSaveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData, EntityImportSettings importSettings) { + protected RuleChain saveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData) { ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting if (existingRuleChain != null) { ruleChainService.deleteRuleNodes(tenantId, existingRuleChain.getId()); diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index e18f73a065..9862ded617 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -72,19 +72,18 @@ CREATE TABLE IF NOT EXISTS entity_alarm ( CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE ); -CREATE TABLE IF NOT EXISTS asset( - id uuid NOT NULL - CONSTRAINT asset_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - type varchar(255), - external_id uuid, - CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) +CREATE TABLE IF NOT EXISTS asset ( + id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + type varchar(255), + external_id uuid, + CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) ); CREATE TABLE IF NOT EXISTS audit_log ( @@ -130,53 +129,50 @@ CREATE TABLE IF NOT EXISTS component_descriptor ( ); CREATE TABLE IF NOT EXISTS customer ( - id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - address varchar, - address2 varchar, - city varchar(255), - country varchar(255), - email varchar(255), - phone varchar(255), - search_text varchar(255), - state varchar(255), - tenant_id uuid, - title varchar(255), - zip varchar(255), - external_id uuid -); - -CREATE TABLE IF NOT EXISTS dashboard -( - id uuid NOT NULL - CONSTRAINT dashboard_pkey PRIMARY KEY, - created_time bigint NOT NULL, - configuration varchar, + id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + address varchar, + address2 varchar, + city varchar(255), + country varchar(255), + email varchar(255), + phone varchar(255), + search_text varchar(255), + state varchar(255), + tenant_id uuid, + title varchar(255), + zip varchar(255), + external_id uuid +); + +CREATE TABLE IF NOT EXISTS dashboard ( + id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY, + created_time bigint NOT NULL, + configuration varchar, assigned_customers varchar(1000000), - search_text varchar(255), - tenant_id uuid, - title varchar(255), - mobile_hide boolean DEFAULT false, - mobile_order int, - image varchar(1000000), - external_id uuid + search_text varchar(255), + tenant_id uuid, + title varchar(255), + mobile_hide boolean DEFAULT false, + mobile_order int, + image varchar(1000000), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_chain ( - id uuid NOT NULL - CONSTRAINT rule_chain_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - configuration varchar(10000000), - name varchar(255), - type varchar(255), - first_rule_node_id uuid, - root boolean, - debug_mode boolean, - search_text varchar(255), - tenant_id uuid, - external_id uuid + id uuid NOT NULL CONSTRAINT rule_chain_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + configuration varchar(10000000), + name varchar(255), + type varchar(255), + first_rule_node_id uuid, + root boolean, + debug_mode boolean, + search_text varchar(255), + tenant_id uuid, + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node ( @@ -224,31 +220,31 @@ CREATE TABLE IF NOT EXISTS ota_package ( ); CREATE TABLE IF NOT EXISTS device_profile ( - id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, - created_time bigint NOT NULL, - name varchar(255), - type varchar(255), - image varchar(1000000), - transport_type varchar(255), - provision_type varchar(255), - profile_data jsonb, - description varchar, - search_text varchar(255), - is_default boolean, - tenant_id uuid, - firmware_id uuid, - software_id uuid, - default_rule_chain_id uuid, - default_dashboard_id uuid, - default_queue_name varchar(255), - provision_device_key varchar, - external_id uuid, - CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), - CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain (id), - CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard (id), - CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package (id), - CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package (id) + id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, + created_time bigint NOT NULL, + name varchar(255), + type varchar(255), + image varchar(1000000), + transport_type varchar(255), + provision_type varchar(255), + profile_data jsonb, + description varchar, + search_text varchar(255), + is_default boolean, + tenant_id uuid, + firmware_id uuid, + software_id uuid, + default_rule_chain_id uuid, + default_dashboard_id uuid, + default_queue_name varchar(255), + provision_device_key varchar, + external_id uuid, + CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), + CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), + CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id), + CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package(id), + CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) ); ALTER TABLE ota_package @@ -266,24 +262,24 @@ ALTER TABLE ota_package -- ); CREATE TABLE IF NOT EXISTS device ( - id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - device_profile_id uuid NOT NULL, - device_data jsonb, - type varchar(255), - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - firmware_id uuid, - software_id uuid, - external_id uuid, - CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile (id), - CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package (id), - CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package (id) + id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + device_profile_id uuid NOT NULL, + device_data jsonb, + type varchar(255), + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + firmware_id uuid, + software_id uuid, + external_id uuid, + CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), + CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), + CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package(id) ); CREATE TABLE IF NOT EXISTS device_credentials ( From 19a27f93f456266895af6026cc86f5c7d27971a6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 31 Mar 2022 19:41:25 +0300 Subject: [PATCH 009/156] Export/import API improvements and refactoring --- .../EntitiesExportImportController.java | 275 ++++++++++++------ .../DefaultEntitiesExportImportService.java | 8 +- .../EntitiesExportImportService.java | 2 +- .../exporting/impl/AssetExportService.java | 2 +- ...vice.java => BaseEntityExportService.java} | 3 +- .../exporting/impl/CustomerExportService.java | 2 +- .../impl/DashboardExportService.java | 2 +- .../exporting/impl/DeviceExportService.java | 3 +- .../impl/DeviceProfileExportService.java | 2 +- .../impl/RuleChainExportService.java | 3 +- .../importing/EntityImportSettings.java | 3 +- .../importing/impl/AssetImportService.java | 8 +- ...vice.java => BaseEntityImportService.java} | 65 +++-- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 46 ++- .../importing/impl/DeviceImportService.java | 8 +- .../impl/DeviceProfileImportService.java | 8 +- .../impl/RuleChainImportService.java | 57 ++-- .../query/DefaultEntityQueryRepository.java | 2 + .../dao/sql/query/EntityKeyMapping.java | 1 + 20 files changed, 320 insertions(+), 184 deletions(-) rename application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/{AbstractEntityExportService.java => BaseEntityExportService.java} (93%) rename application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/{AbstractEntityImportService.java => BaseEntityImportService.java} (68%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index f7513eb2d5..411a5ec00e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections.CollectionUtils; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; @@ -27,10 +25,11 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -38,169 +37,263 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.exportimport.ExportableEntitiesService; import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; -import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @RestController @RequestMapping("/api/entities") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") @TbCoreComponent @RequiredArgsConstructor public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; - private final EntityQueryService entityQueryService; + private final ExportableEntitiesService exportableEntitiesService; + private final EntityService entityService; - // TODO [viacheslav]: export and import of batches - // TODO [viacheslav]: api to export and import whole customer, whole tenant - + @PostMapping("/export/{entityType}/{id}") + public EntityExportData> exportSingleEntity(@PathVariable EntityType entityType, + @PathVariable UUID id, + @RequestParam Map exportSettingsParams) throws ThingsboardException { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); + try { + return exportEntity(getTenantId(), entityId, toExportSettings(exportSettingsParams)); + } catch (Exception e) { + throw handleException(e); + } + } - @PostMapping("/export/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData exportEntity(@ApiParam(allowableValues = "DEVICE, DEVICE_PROFILE, ASSET, RULE_CHAIN, DASHBOARD, CUSTOMER") - @PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - checkEntityId(entityId, Operation.READ); + @PostMapping(value = "/export/{entityType}", params = "ids") + public List>> exportEntitiesByIds(@PathVariable EntityType entityType, + @RequestParam UUID[] ids, + @RequestParam Map exportSettingsParams) throws ThingsboardException { + List entitiesIds = Arrays.stream(ids) + .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) + .collect(Collectors.toList()); + try { + return exportEntitiesByIds(getTenantId(), entitiesIds, toExportSettings(exportSettingsParams)); + } catch (Exception e) { + throw handleException(e); + } + } - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + @PostMapping(value = "/export/{entityType}", params = "ids") + public List>> exportAllEntitiesByEntityType(@PathVariable EntityType entityType, + @RequestParam Map exportSettingsParams, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "2147483647") int pageSize, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); - try { // FIXME [viacheslav]: check read permission for relation fromId - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + try { + return exportEntitiesByFilter(tenantId, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } } @PostMapping("/exportByFilter") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, // TODO [viacheslav]: exportInboundRelations, exportOutboundRelations - @RequestParam(defaultValue = "false") boolean exportInboundRelations, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME), EntityDataSortOrder.Direction.DESC)); - EntityDataQuery entityDataQuery = new EntityDataQuery(filter, pageLink, List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME)), - Collections.emptyList(), Collections.emptyList()); - - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + public List>> exportEntitiesByFilter(@RequestBody EntityFilter filter, + @RequestParam Map exportSettingsParams, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "2147483647") int pageSize, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); + try { + return exportEntitiesByFilter(tenantId, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); + } catch (Exception e) { + throw handleException(e); + } + } + // FIXME: too aggressive + @PostMapping("/exportByFilters") + public List>> exportAllEntitiesByFilters(@RequestBody List filters, + @RequestParam Map exportSettingsParams, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); try { - // FIXME [viacheslav]: check read permission for relation fromId - return entityQueryService.findEntityDataByQuery(user, entityDataQuery).getData().stream() - .map(EntityData::getEntityId) - .map(entityId -> { - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); - }) - .collect(Collectors.toList()); + List>> exportDataList = new ArrayList<>(); + for (EntityFilter filter : filters) { + exportDataList.addAll(exportEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); + } + return exportDataList; } catch (Exception e) { throw handleException(e); } } @PostMapping("/exportByQuery") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntitiesByQuery(@RequestBody EntityDataQuery query, - @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); -// FIXME [viacheslav]: check read permission for relation fromId + public List>> exportEntitiesByQuery(@RequestBody EntityDataQuery entitiesQuery, + @RequestParam Map exportSettingsParams, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + TenantId tenantId = getTenantId(); + CustomerId customerId = toCustomerId(customerUuid); try { - return entityQueryService.findEntityDataByQuery(user, query).getData().stream() - .map(EntityData::getEntityId) - .map(entityId -> exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings)) - .collect(Collectors.toList()); + return exportEntitiesByQuery(tenantId, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } } - @PostMapping("/import") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> importEntity(@RequestBody List>> exportDataList, - @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - EntityImportSettings importSettings = toImportSettings(importInboundRelations); + private List>> exportEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - for (EntityExportData> exportData : exportDataList) { - checkPermissionsForImport(user, exportData, importSettings); - } + EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + return exportEntitiesByQuery(tenantId, customerId, query, exportSettings); + } - try { - List>> importResultList = exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); - importResultList.forEach(entityImportResult -> { - onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); - }); - return importResultList; - } catch (Exception e) { - throw handleException(e); + private List>> exportEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { + List entitiesIds = entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + .map(EntityData::getEntityId) + .collect(Collectors.toList()); + return exportEntitiesByIds(tenantId, entitiesIds, exportSettings); + } + + private List>> exportEntitiesByIds(TenantId tenantId, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { + List>> exportDataList = new ArrayList<>(); + for (EntityId entityId : entitiesIds) { + exportDataList.add(exportEntity(tenantId, entityId, exportSettings)); } + return exportDataList; } - public void checkPermissionsForExport(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { + private , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); + + List relations = new LinkedList<>(); if (exportSettings.isExportInboundRelations()) { - for (EntityRelation entityRelation : relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON)) { - EntityId fromId = entityRelation.getFrom(); - checkEntityId(fromId, Operation.READ); + relations.addAll(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)); + } + if (exportSettings.isExportOutboundRelations()) { + relations.addAll(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)); + } + for (EntityRelation relation : relations) { + if (!relation.getFrom().equals(entityId)) { + checkEntityId(relation.getFrom(), Operation.READ); + } else if (!relation.getTo().equals(entityId)) { + checkEntityId(relation.getTo(), Operation.READ); } } + + return exportImportService.exportEntity(tenantId, entityId, exportSettings); } - public void checkPermissionsForImport(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { - ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) { - checkEntityId(existingEntity.getId(), Operation.WRITE); - } else { - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + + @PostMapping("/import") + public List>> importEntity(@RequestBody List>> exportDataList, + @RequestParam Map importSettingsParams) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityImportSettings importSettings = toImportSettings(importSettingsParams); + + try { + return importEntities(user, exportDataList, importSettings) + .stream().peek(entityImportResult -> { + onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); + }) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); } + } + - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - for (EntityRelation entityRelation : exportData.getInboundRelations()) { - ExportableEntity entityFrom = exportImportService.findEntityByExternalId(user.getTenantId(), entityRelation.getFrom()); - if (entityFrom != null) { - accessControlService.checkPermission(user, Resource.of(entityFrom.getId().getEntityType()), Operation.WRITE, entityFrom.getId(), entityFrom); - } else { - throw new ThingsboardException("Relation with non-existing entity", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + for (EntityExportData> exportData : exportDataList) { + ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { + checkEntityId(existingEntity.getId(), Operation.WRITE); + } else { + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + } + + List relations = new LinkedList<>(); + if (importSettings.isImportInboundRelations() && exportData.getInboundRelations() != null) { + relations.addAll(exportData.getInboundRelations()); + } + if (importSettings.isImportOutboundRelations() && exportData.getOutboundRelations() != null) { + relations.addAll(exportData.getOutboundRelations()); + } + for (EntityRelation relation : relations) { + EntityId otherEntityId = null; + if (!relation.getFrom().equals(exportData.getMainEntity().getId())) { + otherEntityId = relation.getFrom(); + } else if (!relation.getTo().equals(exportData.getMainEntity().getId())) { + otherEntityId = relation.getTo(); + } + if (otherEntityId != null) { + ExportableEntity otherEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), otherEntityId); + if (otherEntity != null) { + checkEntityId(otherEntity.getId(), Operation.WRITE); + } else { + throw new IllegalArgumentException("Relation is referencing non-existing entity"); + } } } } + + return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); } - private EntityImportSettings toImportSettings(boolean importInboundRelations) { - return EntityImportSettings.builder() - .importInboundRelations(importInboundRelations) + private EntityExportSettings toExportSettings(Map exportSettingsParams) { + return EntityExportSettings.builder() + .exportInboundRelations(getParam(exportSettingsParams, "exportInboundRelations", false, Boolean::parseBoolean)) + .exportOutboundRelations(getParam(exportSettingsParams, "exportOutboundRelations", false, Boolean::parseBoolean)) .build(); } - private EntityExportSettings toExportSettings(boolean exportInboundRelations) { - return EntityExportSettings.builder() - .exportInboundRelations(exportInboundRelations) + private EntityImportSettings toImportSettings(Map importSettingsParams) { + return EntityImportSettings.builder() + .importInboundRelations(getParam(importSettingsParams, "importInboundRelations", false, Boolean::parseBoolean)) + .importOutboundRelations(getParam(importSettingsParams, "importOutboundRelations", false, Boolean::parseBoolean)) + .removeExistingRelations(getParam(importSettingsParams, "removeExistingRelations", true, Boolean::parseBoolean)) + .updateReferencesToOtherEntities(getParam(importSettingsParams, "updateReferencesToOtherEntities", true, Boolean::parseBoolean)) .build(); } + protected T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { + return parsingFunction.apply(requestParams.getOrDefault(key, defaultValue.toString())); + } + + private CustomerId toCustomerId(UUID customerUuid) { + return new CustomerId(Optional.ofNullable(customerUuid).orElse(EntityId.NULL_UUID)); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 93fa370c94..8d85355849 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -39,10 +39,9 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -// FIXME [viacheslav]: review packages and classes naming @Service @TbCoreComponent -public class DefaultEntitiesExportImportService implements EntitiesExportImportService { +public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); @@ -54,8 +53,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS ); - // TODO [viacheslav]: export and import of the whole tenant - // TODO [viacheslav]: export and import of the whole customer ? @Override public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { EntityType entityType = entityId.getEntityType(); @@ -65,7 +62,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - // FIXME [viacheslav]: somehow validate export data + // TODO [viacheslav]: validate export data @Transactional @Override public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { @@ -80,6 +77,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { return exportDataList.stream() .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) + // TODO [viacheslav]: order for rule chains (depending on references) .map(exportData -> importEntity(tenantId, exportData, importSettings)) .collect(Collectors.toList()); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index e8d4a40efe..682db6ba21 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -25,7 +25,7 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportSetting import java.util.List; -public interface EntitiesExportImportService extends ExportableEntitiesService { +public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java index a48167053b..c22b345299 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.AssetExportDat @Service @TbCoreComponent -public class AssetExportService extends AbstractEntityExportService { +public class AssetExportService extends BaseEntityExportService { @Override protected AssetExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java index ee6437c7dc..68c78e87c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AbstractEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java @@ -30,14 +30,13 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa import java.util.List; -public abstract class AbstractEntityExportService, D extends EntityExportData> implements EntityExportService { +public abstract class BaseEntityExportService, D extends EntityExportData> implements EntityExportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; - @Override public final D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { D exportData = newExportData(); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java index 518a830a21..1f23e8e7d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.CustomerExport @Service @TbCoreComponent -public class CustomerExportService extends AbstractEntityExportService { +public class CustomerExportService extends BaseEntityExportService { @Override protected CustomerExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java index 167b57be75..17d593157b 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.DashboardExpor @Service @TbCoreComponent -public class DashboardExportService extends AbstractEntityExportService { +public class DashboardExportService extends BaseEntityExportService { @Override protected DashboardExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java index 8a8bc26820..b8c7e65b34 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java @@ -28,11 +28,10 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportDa @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceExportService extends AbstractEntityExportService { +public class DeviceExportService extends BaseEntityExportService { private final DeviceCredentialsService deviceCredentialsService; - @Override protected void setRelatedEntities(TenantId tenantId, Device device, DeviceExportData exportData) { exportData.setCredentials(deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, device.getId())); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java index ce96e3b1fe..24b5f8891e 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java @@ -24,7 +24,7 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileE @Service @TbCoreComponent -public class DeviceProfileExportService extends AbstractEntityExportService { +public class DeviceProfileExportService extends BaseEntityExportService { @Override protected DeviceProfileExportData newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java index 8c173fe2ce..f1b9a27f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java @@ -28,11 +28,10 @@ import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExpor @Service @TbCoreComponent @RequiredArgsConstructor -public class RuleChainExportService extends AbstractEntityExportService { +public class RuleChainExportService extends BaseEntityExportService { private final RuleChainService ruleChainService; - @Override protected void setRelatedEntities(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData) { exportData.setMetaData(ruleChainService.loadRuleChainMetaData(tenantId, ruleChain.getId())); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java index 22b67bcbb4..c3cf0b531d 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java @@ -27,7 +27,6 @@ import lombok.NoArgsConstructor; public class EntityImportSettings { private boolean importInboundRelations; private boolean importOutboundRelations; - private boolean removeExistingRelationsAndSaveNew; - + private boolean removeExistingRelations; private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java index 0d6e6d01b5..04e5a5eeca 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java @@ -28,17 +28,13 @@ import org.thingsboard.server.service.exportimport.exporting.data.AssetExportDat @Service @TbCoreComponent @RequiredArgsConstructor -public class AssetImportService extends AbstractEntityImportService { +public class AssetImportService extends BaseEntityImportService { private final AssetService assetService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, Asset asset, IdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, AssetExportData exportData, NewIdProvider idProvider) { asset.setCustomerId(idProvider.get(tenantId, Asset::getCustomerId)); - } - - @Override - protected Asset saveEntity(TenantId tenantId, Asset asset, Asset existingAsset, AssetExportData exportData) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java similarity index 68% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java index b521074bab..214e314ffd 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AbstractEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java @@ -15,10 +15,12 @@ */ package org.thingsboard.server.service.exportimport.importing.impl; +import lombok.RequiredArgsConstructor; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -32,12 +34,16 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportService; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.stream.Collectors; -public abstract class AbstractEntityImportService, D extends EntityExportData> implements EntityImportService { +public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy private ExportableEntitiesService exportableEntitiesService; @@ -59,18 +65,7 @@ public abstract class AbstractEntityImportService() { - @Override - public ID get(TenantId tenantId, Function idExtractor) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { - return getInternalId(tenantId, idExtractor.apply(entity)); - } else { - return idExtractor.apply(existingEntity); - } - } - }); - - E savedEntity = saveEntity(tenantId, entity, existingEntity, exportData); + E savedEntity = prepareAndSave(tenantId, entity, exportData, new NewIdProvider(entity, existingEntity, importSettings)); importRelations(tenantId, savedEntity, existingEntity, exportData, importSettings); EntityImportResult importResult = new EntityImportResult<>(); @@ -79,9 +74,7 @@ public abstract class AbstractEntityImportService idProvider) {} - - protected abstract E saveEntity(TenantId tenantId, E entity, E existingEntity, D exportData); + protected abstract E prepareAndSave(TenantId tenantId, E entity, D exportData, NewIdProvider idProvider); private void importRelations(TenantId tenantId, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) { @@ -94,7 +87,7 @@ public abstract class AbstractEntityImportService { relationService.deleteRelation(tenantId, existingRelation); }); @@ -107,14 +100,16 @@ public abstract class AbstractEntityImportService { relationService.deleteRelation(tenantId, existingRelation); }); } } - newRelations.forEach(relation -> relationService.saveRelation(tenantId, relation)); + newRelations.forEach(relation -> { + relationService.saveRelation(tenantId, relation); + }); } private ID getInternalId(TenantId tenantId, ID externalId) { @@ -128,8 +123,36 @@ public abstract class AbstractEntityImportService { - I get(TenantId tenantId, Function idExtractor); + @RequiredArgsConstructor + protected class NewIdProvider { + private final E entity; + private final E existingEntity; + private final EntityImportSettings importSettings; + + private final Set ALWAYS_UPDATE_REFERENCED_IDS = Set.of( + EntityType.RULE_CHAIN + ); + + public ID get(TenantId tenantId, Function idExtractor) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities() + || ALWAYS_UPDATE_REFERENCED_IDS.contains(getEntityType())) { + return getInternalId(tenantId, idExtractor.apply(entity)); + } else { + return idExtractor.apply(existingEntity); + } + } + + public Set get(TenantId tenantId, Function> listExtractor, Function idGetter, BiConsumer idSetter) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { + return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() + .peek(t -> { + idSetter.accept(t, getInternalId(tenantId, idGetter.apply(t))); + }) + .collect(Collectors.toSet()); + } else { + return listExtractor.apply(existingEntity); + } + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java index 22cdc10204..7b5b3e3248 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java @@ -28,12 +28,12 @@ import org.thingsboard.server.service.exportimport.exporting.data.CustomerExport @Service @TbCoreComponent @RequiredArgsConstructor -public class CustomerImportService extends AbstractEntityImportService { +public class CustomerImportService extends BaseEntityImportService { private final CustomerService customerService; @Override - protected Customer saveEntity(TenantId tenantId, Customer customer, Customer existingCustomer, CustomerExportData exportData) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, CustomerExportData exportData, NewIdProvider idProvider) { return customerService.saveCustomer(customer); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java index d78f0b40f5..2bf9a7926a 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java @@ -19,31 +19,55 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; + @Service @TbCoreComponent @RequiredArgsConstructor -public class DashboardImportService extends AbstractEntityImportService { +public class DashboardImportService extends BaseEntityImportService { private final DashboardService dashboardService; + // TODO [viacheslav]: improve the code @Override - protected void setLinkedEntitiesIds(TenantId tenantId, Dashboard dashboard, IdProvider idProvider) { -// if (existingDashboard == null) { -// dashboard.setAssignedCustomers(null); // FIXME [viacheslav]: need to assign dashboard to customers ? -// } else { -// dashboard.setAssignedCustomers(existingDashboard.getAssignedCustomers()); -// } - } + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, DashboardExportData exportData, NewIdProvider idProvider) { + if (dashboard.getId() == null) { + Set assignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); + dashboard.setAssignedCustomers(null); + dashboard = dashboardService.saveDashboard(dashboard); + for (ShortCustomerInfo customerInfo : assignedCustomers) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerInfo.getCustomerId()); + } + } else { + Set existingAssignedCustomers = dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers().stream() + .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set newAssignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() + .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); - @Override - protected Dashboard saveEntity(TenantId tenantId, Dashboard dashboard, Dashboard existingDashboard, DashboardExportData exportData) { - return dashboardService.saveDashboard(dashboard); + Set toUnassign = new HashSet<>(existingAssignedCustomers); + toUnassign.removeAll(newAssignedCustomers); + for (CustomerId customerId : toUnassign) { + dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId); + } + Set toAssign = new HashSet<>(newAssignedCustomers); + toAssign.removeAll(existingAssignedCustomers); + for (CustomerId customerId : toAssign) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + } + dashboard.setAssignedCustomers(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()); + dashboard = dashboardService.saveDashboard(dashboard); + } + return dashboard; } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java index 8c02edec05..ee6d89b774 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java @@ -28,20 +28,16 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportDa @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceImportService extends AbstractEntityImportService { +public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, Device device, IdProvider idProvider) { + protected Device prepareAndSave(TenantId tenantId, Device device, DeviceExportData exportData, NewIdProvider idProvider) { device.setCustomerId(idProvider.get(tenantId, Device::getCustomerId)); device.setDeviceProfileId(idProvider.get(tenantId, Device::getDeviceProfileId)); device.setFirmwareId(idProvider.get(tenantId, Device::getFirmwareId)); device.setSoftwareId(idProvider.get(tenantId, Device::getSoftwareId)); - } - - @Override - protected Device saveEntity(TenantId tenantId, Device device, Device existingDevice, DeviceExportData exportData) { return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java index 9981607481..5d05bd06eb 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java @@ -28,20 +28,16 @@ import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileE @Service @TbCoreComponent @RequiredArgsConstructor -public class DeviceProfileImportService extends AbstractEntityImportService { +public class DeviceProfileImportService extends BaseEntityImportService { private final DeviceProfileService deviceProfileService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) { + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfileExportData exportData, NewIdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.get(tenantId, DeviceProfile::getDefaultRuleChainId)); deviceProfile.setDefaultDashboardId(idProvider.get(tenantId, DeviceProfile::getDefaultDashboardId)); deviceProfile.setFirmwareId(idProvider.get(tenantId, DeviceProfile::getFirmwareId)); deviceProfile.setSoftwareId(idProvider.get(tenantId, DeviceProfile::getSoftwareId)); - } - - @Override - protected DeviceProfile saveEntity(TenantId tenantId, DeviceProfile deviceProfile, DeviceProfile existingDeviceProfile, DeviceProfileExportData exportData) { return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java index 6be2fc4317..34d346ccc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java @@ -15,6 +15,9 @@ */ package org.thingsboard.server.service.exportimport.importing.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; @@ -26,39 +29,47 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import java.util.Collections; +import java.util.Optional; +import java.util.UUID; + @Service @TbCoreComponent @RequiredArgsConstructor -public class RuleChainImportService extends AbstractEntityImportService { +public class RuleChainImportService extends BaseEntityImportService { private final RuleChainService ruleChainService; @Override - protected void setLinkedEntitiesIds(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) { - } + protected RuleChain prepareAndSave(TenantId tenantId, RuleChain ruleChain, RuleChainExportData exportData, NewIdProvider idProvider) { + RuleChainMetaData metaData = exportData.getMetaData(); + Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()) + .forEach(ruleNode -> { + ruleNode.setId(null); + ruleNode.setRuleChainId(null); + JsonNode ruleNodeConfig = ruleNode.getConfiguration(); + Optional.ofNullable(ruleNodeConfig) + .flatMap(config -> Optional.ofNullable(config.get("ruleChainId")).filter(JsonNode::isTextual)) + .map(JsonNode::asText).map(UUID::fromString) + .ifPresent(otherRuleChainUuid -> { + ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( + idProvider.get(tenantId, rc -> new RuleChainId(otherRuleChainUuid)).toString() + )); + ruleNode.setConfiguration(ruleNodeConfig); + }); + }); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(tenantId, rc -> ruleChainConnectionInfo.getTargetRuleChainId())); + }); + ruleChain.setFirstRuleNodeId(null); - @Override - protected RuleChain saveEntity(TenantId tenantId, RuleChain ruleChain, RuleChain existingRuleChain, RuleChainExportData exportData) { - ruleChain.setFirstRuleNodeId(null); // will be set during metadata persisting - if (existingRuleChain != null) { - ruleChainService.deleteRuleNodes(tenantId, existingRuleChain.getId()); + if (ruleChain.getId() != null) { + ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); } - ruleChain = ruleChainService.saveRuleChain(ruleChain); - - RuleChainMetaData metaData = exportData.getMetaData(); - metaData.setRuleChainId(ruleChain.getId()); - metaData.getNodes().forEach(ruleNode -> { - ruleNode.setId(null); - ruleNode.setRuleChainId(null); - }); - metaData.getRuleChainConnections().forEach(ruleChainConnectionInfo -> { -// ruleChainConnectionInfo.setTargetRuleChainId(); - // TODO [viacheslav]: check if this thing is needed for "Other Rule Chain Node" - // TODO [viacheslav]: and check import of tenant rule chains - }); - ruleChainService.saveRuleChainMetaData(tenantId, metaData); - + exportData.getMetaData().setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(tenantId, exportData.getMetaData()); return ruleChain; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index f4fa052648..47ce08c4c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -238,6 +238,8 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { entityTableMap.put(EntityType.TENANT, "tenant"); entityTableMap.put(EntityType.API_USAGE_STATE, SELECT_API_USAGE_STATE); entityTableMap.put(EntityType.EDGE, "edge"); + entityTableMap.put(EntityType.RULE_CHAIN, "rule_chain"); + entityTableMap.put(EntityType.DEVICE_PROFILE, "device_profile"); } public static EntityType[] RELATION_QUERY_ENTITY_TYPES = new EntityType[]{ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index 8df341fb0b..1499b7150f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -102,6 +102,7 @@ public class EntityKeyMapping { allowedEntityFieldMap.put(EntityType.WIDGET_TYPE, new HashSet<>(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.WIDGETS_BUNDLE, new HashSet<>(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.API_USAGE_STATE, apiUsageStateEntityFields); + allowedEntityFieldMap.put(EntityType.DEVICE_PROFILE, Set.of(CREATED_TIME, NAME, TYPE)); entityFieldColumnMap.put(CREATED_TIME, ModelConstants.CREATED_TIME_PROPERTY); entityFieldColumnMap.put(ENTITY_TYPE, ModelConstants.ENTITY_TYPE_PROPERTY); From e37a075031ea450c8777048949815062b23b0687 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 1 Apr 2022 15:06:50 +0300 Subject: [PATCH 010/156] Export/import API: refactor, rename packages --- .../server/controller/AssetController.java | 4 +-- .../server/controller/BaseController.java | 12 ++++--- .../server/controller/DeviceController.java | 4 +-- .../server/controller/EdgeController.java | 4 +-- .../EntitiesExportImportController.java | 35 ++++++++++--------- .../controller/RuleChainController.java | 1 - .../service/asset/AssetBulkImportService.java | 4 +-- .../device/DeviceBulkImportService.java | 4 +-- .../service/edge/EdgeBulkImportService.java | 4 +-- .../DefaultEntitiesExportImportService.java | 15 ++++---- .../EntitiesExportImportService.java | 10 +++--- .../exporting/EntityExportService.java | 4 +-- .../exporting/EntityExportSettings.java | 2 +- .../exporting}/ExportableEntitiesService.java | 2 +- .../exporting/data/AssetExportData.java | 2 +- .../exporting/data/CustomerExportData.java | 2 +- .../exporting/data/DashboardExportData.java | 2 +- .../exporting/data/DeviceExportData.java | 2 +- .../data/DeviceProfileExportData.java | 2 +- .../exporting/data/EntityExportData.java | 6 ++-- .../exporting/data/RuleChainExportData.java | 2 +- .../exporting/impl/AssetExportService.java | 4 +-- .../impl/BaseEntityExportService.java | 16 ++++----- .../exporting/impl/CustomerExportService.java | 4 +-- .../impl/DashboardExportService.java | 4 +-- .../exporting/impl/DeviceExportService.java | 4 +-- .../impl/DeviceProfileExportService.java | 4 +-- .../impl/RuleChainExportService.java | 4 +-- .../importing/EntityImportResult.java | 2 +- .../importing/EntityImportService.java | 4 +-- .../importing/EntityImportSettings.java | 2 +- .../csv}/AbstractBulkImportService.java | 4 +-- .../importing/csv}/BulkImportColumnType.java | 2 +- .../importing/csv}/BulkImportRequest.java | 2 +- .../importing/csv}/BulkImportResult.java | 2 +- .../importing/csv}/ImportedEntityInfo.java | 2 +- .../importing/impl/AssetImportService.java | 4 +-- .../impl/BaseEntityImportService.java | 16 ++++----- .../importing/impl/CustomerImportService.java | 4 +-- .../impl/DashboardImportService.java | 4 +-- .../importing/impl/DeviceImportService.java | 4 +-- .../impl/DeviceProfileImportService.java | 4 +-- .../impl/RuleChainImportService.java | 6 ++-- .../server/dao/rule/RuleChainService.java | 1 + .../server/common/data/id/IdBased.java | 35 ++++++++++--------- .../server/common/data/rule/RuleChain.java | 4 +-- .../server/dao/ExportableEntityDao.java | 1 - 47 files changed, 136 insertions(+), 130 deletions(-) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/DefaultEntitiesExportImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/EntitiesExportImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/EntityExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/EntityExportSettings.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync/exporting}/ExportableEntitiesService.java (95%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/AssetExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/CustomerExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/DashboardExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/DeviceExportData.java (94%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/DeviceProfileExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/EntityExportData.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/data/RuleChainExportData.java (94%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/AssetExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/BaseEntityExportService.java (79%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/CustomerExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/DashboardExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/DeviceProfileExportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/EntityImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/EntityImportService.java (88%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/EntityImportSettings.java (94%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/AbstractBulkImportService.java (98%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/BulkImportColumnType.java (97%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/BulkImportRequest.java (94%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/BulkImportResult.java (94%) rename application/src/main/java/org/thingsboard/server/service/{importing => sync/importing/csv}/ImportedEntityInfo.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/AssetImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/BaseEntityImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/CustomerImportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/DeviceImportService.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/DeviceProfileImportService.java (92%) rename application/src/main/java/org/thingsboard/server/service/{exportimport => sync}/importing/impl/RuleChainImportService.java (94%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 94c714e24f..b05e32efe7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -54,8 +54,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 7416444104..382929e1e7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -924,7 +924,7 @@ public abstract class BaseController { } } - protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); @@ -951,7 +951,7 @@ public abstract class BaseController { otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); notifyEdgeService = true; break; - case RULE_CHAIN: // FIXME: events for rule chain metadata + case RULE_CHAIN: RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); if (RuleChainType.CORE.equals(ruleChainType)) { tbClusterService.broadcastEntityStateChangeEvent(savedEntity.getTenantId(), savedEntity.getId(), @@ -974,8 +974,12 @@ public abstract class BaseController { throw new UnsupportedOperationException(); } - entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, - isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); + try { + logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, + isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); + } catch (ThingsboardException e) { + log.error("Failed to log entity action", e); + } if (notifyEdgeService) { sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index d2ffafa73e..feb58edb7a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,8 +74,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index f3d73f13e5..8f45427603 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -55,8 +55,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 411a5ec00e..f158ff15e7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -42,12 +42,12 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.EntitiesExportImportService; -import org.thingsboard.server.service.exportimport.ExportableEntitiesService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -103,12 +103,12 @@ public class EntitiesExportImportController extends BaseController { } } - @PostMapping(value = "/export/{entityType}", params = "ids") - public List>> exportAllEntitiesByEntityType(@PathVariable EntityType entityType, - @RequestParam Map exportSettingsParams, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "2147483647") int pageSize, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { + @PostMapping(value = "/export/{entityType}") + public List>> exportEntitiesByEntityType(@PathVariable EntityType entityType, + @RequestParam Map exportSettingsParams, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "2147483647") int pageSize, + @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { TenantId tenantId = getTenantId(); CustomerId customerId = toCustomerId(customerUuid); @@ -236,11 +236,12 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { for (EntityExportData> exportData : exportDataList) { - ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getEntity().getId()); if (existingEntity != null) { - checkEntityId(existingEntity.getId(), Operation.WRITE); + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.WRITE, existingEntity.getId(), existingEntity); } else { - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + exportData.getEntity().setTenantId(user.getTenantId()); + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE, null, exportData.getEntity()); } List relations = new LinkedList<>(); @@ -252,9 +253,9 @@ public class EntitiesExportImportController extends BaseController { } for (EntityRelation relation : relations) { EntityId otherEntityId = null; - if (!relation.getFrom().equals(exportData.getMainEntity().getId())) { + if (!relation.getFrom().equals(exportData.getEntity().getId())) { otherEntityId = relation.getFrom(); - } else if (!relation.getTo().equals(exportData.getMainEntity().getId())) { + } else if (!relation.getTo().equals(exportData.getEntity().getId())) { otherEntityId = relation.getTo(); } if (otherEntityId != null) { diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index aba074ac60..cb59951ac5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -279,7 +279,6 @@ public class RuleChainController extends BaseController { RuleChain savedRuleChain = installScripts.createDefaultRuleChain(getCurrentUser().getTenantId(), request.getName()); - tbClusterService.broadcastEntityStateChangeEvent(savedRuleChain.getTenantId(), savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); logEntityAction(savedRuleChain.getId(), savedRuleChain, null, ActionType.ADDED, null); diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index 4c75f7d45d..cfefcafbf4 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; +import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index cd655a2467..3aac9c3641 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; +import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index 615a7cddd2..b09ac532df 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; +import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 8d85355849..f399dbdbea 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport; +package org.thingsboard.server.service.sync; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -24,12 +24,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportService; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.Collection; import java.util.Comparator; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 682db6ba21..0fbe75a866 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport; +package org.thingsboard.server.service.sync; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index 929092e90e..6b57cbb557 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting; +package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java index f4eea9913a..36552b2c12 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting; +package org.thingsboard.server.service.sync.exporting; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index bb8f30c4e1..2135d2aca1 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport; +package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java index 4c4872760b..4031ec7062 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/AssetExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java index 1fc820814e..08888f4a35 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/CustomerExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java index 6ef45fc531..6f101816c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DashboardExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java index 5f5d8e8f72..977bd22f82 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java index def6b7edbe..7b71749be4 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/DeviceProfileExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index b9cc8c71a3..51b5615c9e 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(property = "entityType", use = JsonTypeInfo.Id.NAME) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType") @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), @@ -43,7 +43,7 @@ import java.util.List; @Data public abstract class EntityExportData> { - private E mainEntity; + private E entity; private List inboundRelations; private List outboundRelations; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java index f92616fead..9f59aa4401 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.data; +package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java index c22b345299..b4f506a1f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/AssetExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; +import org.thingsboard.server.service.sync.exporting.data.AssetExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 68c78e87c3..a3a83196bb 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -23,10 +23,10 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.exportimport.ExportableEntitiesService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import java.util.List; @@ -41,9 +41,9 @@ public abstract class BaseEntityExportService inboundRelations = relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java index 1f23e8e7d2..c90d21e573 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/CustomerExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; +import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java index 17d593157b..32584ca917 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java index b8c7e65b34..d31a859013 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java index 24b5f8891e..b3fd65363b 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/DeviceProfileExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java index f1b9a27f23..846d4006d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java index cc44670757..a79bfa2478 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing; +package org.thingsboard.server.service.sync.importing; import lombok.Data; import org.thingsboard.server.common.data.ExportableEntity; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java similarity index 88% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java index 8697b04f59..19947e8917 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing; +package org.thingsboard.server.service.sync.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java index c3cf0b531d..9e4afcc560 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing; +package org.thingsboard.server.service.sync.importing; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java similarity index 98% rename from application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java index b7b740f190..06cd7a93be 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,7 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.importing.BulkImportRequest.ColumnMapping; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java index 96075eb4c8..beafe5e95f 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java index 9f8195ca45..f600289ab6 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java index 651aedeb0b..da12a6baeb 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java index 45c2551be2..68e7a49b18 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 04e5a5eeca..b8328a43c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.AssetExportData; +import org.thingsboard.server.service.sync.exporting.data.AssetExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 214e314ffd..608ffe2acf 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.apache.commons.collections.CollectionUtils; @@ -28,11 +28,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.exportimport.ExportableEntitiesService; -import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.exportimport.importing.EntityImportResult; -import org.thingsboard.server.service.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportService; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.Collections; import java.util.LinkedList; @@ -53,7 +53,7 @@ public abstract class BaseEntityImportService importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { - E entity = exportData.getMainEntity(); + E entity = exportData.getEntity(); E existingEntity = exportableEntitiesService.findEntityByExternalId(tenantId, entity.getId()); entity.setExternalId(entity.getId()); @@ -118,7 +118,7 @@ public abstract class BaseEntityImportService entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); if (entity == null) { - throw new IllegalStateException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); + throw new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); } return entity.getId(); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index 7b5b3e3248..5ad7161486 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.CustomerExportData; +import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 2bf9a7926a..9e7cffd272 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; import java.util.HashSet; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index ee6d89b774..eafe9c0fdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index 5d05bd06eb..61b1ea16dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 34d346ccc9..5dfc31e97a 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.exportimport.importing.impl; +package org.thingsboard.server.service.sync.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; import java.util.Collections; import java.util.Optional; @@ -70,7 +70,7 @@ public class RuleChainImportService extends BaseEntityImportService implements HasId { protected I id; - - public IdBased() { - super(); - } - - public IdBased(I id) { - super(); - this.id = id; - } + + public IdBased() { + super(); + } + + public IdBased(I id) { + super(); + this.id = id; + } @JsonSetter - public void setId(I id) { - this.id = id; - } + public void setId(I id) { + this.id = id; + } - public I getId() { - return id; - } + public I getId() { + return id; + } - @JsonIgnore - public UUID getUuidId() { + @JsonIgnore + public UUID getUuidId() { if (id != null) { return id.getId(); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 74186b49bf..eca9e331cd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -57,11 +57,11 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im @ApiModelProperty(position = 9, value = "Reserved for future usage. The actual list of rule nodes and their relations is stored in the database separately.") private transient JsonNode configuration; + private RuleChainId externalId; + @JsonIgnore private byte[] configurationBytes; - private RuleChainId externalId; - public RuleChain() { super(); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 2fb301c092..5b2b5a2366 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -25,7 +25,6 @@ public interface ExportableEntityDao { T findByTenantIdAndId(UUID tenantId, UUID id); - EntityType getEntityType(); } From 4dcd037a61eeadf576a59d3e8275596b3699a612 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 1 Apr 2022 15:24:22 +0300 Subject: [PATCH 011/156] Initial tests for export/import --- .../server/controller/AbstractWebTest.java | 1 - ...aseEntitiesExportImportControllerTest.java | 166 +++++++++++++ ...EntitiesExportImportControllerSqlTest.java | 228 ++++++++++++++++++ 3 files changed, 394 insertions(+), 1 deletion(-) create mode 100644 application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java create mode 100644 application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index cc561e31a8..04a2c5f351 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -605,7 +605,6 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected T readResponse(ResultActions result, TypeReference type) throws Exception { byte[] content = result.andReturn().getResponse().getContentAsByteArray(); - ObjectMapper mapper = new ObjectMapper(); return mapper.readerFor(type).readValue(content); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java new file mode 100644 index 0000000000..cf88cf3d0d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.fasterxml.jackson.databind.node.TextNode; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.rule.RuleChainService; + +import java.util.Arrays; +import java.util.Collections; + +public abstract class BaseEntitiesExportImportControllerTest extends AbstractControllerTest { + + @Autowired + protected DeviceService deviceService; + @Autowired + protected DeviceProfileService deviceProfileService; + @Autowired + protected AssetService assetService; + @Autowired + protected CustomerService customerService; + @Autowired + protected RuleChainService ruleChainService; + @Autowired + protected DashboardService dashboardService; + + protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName(name); + device.setLabel("lbl"); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + device.setDeviceData(deviceData); + return deviceService.saveDevice(device); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, String name) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setName(name); + deviceProfile.setDescription("dscrptn"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setCustomerId(customerId); + asset.setType(type); + asset.setName(name); + asset.setLabel("lbl"); + asset.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return assetService.saveAsset(asset); + } + + protected Customer createCustomer(TenantId tenantId, String name) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(name); + customer.setCountry("ua"); + customer.setAddress("abb"); + customer.setEmail("ccc@aa.org"); + customer.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return customerService.saveCustomer(customer); + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenantId); + dashboard.setTitle(name); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + dashboard.setImage("abvregewrg"); + dashboard.setMobileHide(true); + dashboard = dashboardService.saveDashboard(dashboard); + if (customerId != null) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + return dashboardService.findDashboardById(tenantId, dashboard.getId()); + } + return dashboard; + } + + protected RuleChain createRuleChain(TenantId tenantId, String name) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java new file mode 100644 index 0000000000..d986f5f9ad --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -0,0 +1,228 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller.sql; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JavaType; +import lombok.SneakyThrows; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.web.servlet.ResultActions; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.controller.BaseEntitiesExportImportControllerTest; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; + +import java.util.List; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { + + @Autowired + private TenantService tenantService; + @Autowired + private DeviceCredentialsService deviceCredentialsService; + @Autowired + private ExportableEntitiesService exportableEntitiesService; + + private TenantId tenantId1; + private User tenantAdmin1; + + private TenantId tenantId2; + private User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } + + @Test + public void testExportImport_singleEntityOneByOne_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); + testExportImportBetweenTenants(asset, importedAsset -> { + assertThat(importedAsset.getName()).isEqualTo(asset.getName()); + assertThat(importedAsset.getType()).isEqualTo(asset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(asset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(asset.getAdditionalInfo()); + }); + + Customer customer = createCustomer(tenantId1, "Customer of tenant 1"); + testExportImportBetweenTenants(customer, importedCustomer -> { + assertThat(importedCustomer.getTitle()).isEqualTo(customer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(customer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(customer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(customer.getEmail()); + }); + + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); + DeviceProfileId importedDeviceProfileId = testExportImportBetweenTenants(deviceProfile, importedDeviceProfile -> { + assertThat(importedDeviceProfile.getName()).isEqualTo(deviceProfile.getName()); + assertThat(importedDeviceProfile.getType()).isEqualTo(deviceProfile.getType()); + assertThat(importedDeviceProfile.getTransportType()).isEqualTo(deviceProfile.getTransportType()); + assertThat(importedDeviceProfile.getProfileData()).isEqualTo(deviceProfile.getProfileData()); + assertThat(importedDeviceProfile.getDescription()).isEqualTo(deviceProfile.getDescription()); + }).getSavedEntity().getId(); + + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + this.testExportImportBetweenTenants(device, deviceExportData -> { + assertThat(deviceExportData.getCredentials()).isEqualTo(credentials); + }, importedDevice -> { + assertThat(importedDevice.getName()).isEqualTo(device.getName()); + assertThat(importedDevice.getType()).isEqualTo(device.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(device.getDeviceData()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(importedDeviceProfileId); + assertThat(importedDevice.getLabel()).isEqualTo(device.getLabel()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(credentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + }); + + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + this.testExportImportBetweenTenants(ruleChain, ruleChainExportData -> { + assertThat(ruleChainExportData.getMetaData()).isEqualTo(metaData); + }, importedRuleChain -> { + assertThat(importedRuleChain.getType()).isEqualTo(ruleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(ruleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(ruleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(ruleChain.getConfiguration()); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + assertThat(importedMetaData.getConnections()).isEqualTo(metaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(metaData.getFirstNodeIndex()); + for (int i = 0; i < metaData.getNodes().size(); i++) { + RuleNode initialNode = metaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + }); + + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard of tenant 1"); + testExportImportBetweenTenants(dashboard, importedDashboard -> { + assertThat(importedDashboard.getTitle()).isEqualTo(dashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(dashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(dashboard.isMobileHide()); + }); + } + + private , D extends EntityExportData> EntityImportResult testExportImportBetweenTenants(E entity, Consumer exportDataRequirements, Consumer importedEntityRequirements) throws Exception { + logInAsTenantAdmin1(); + D exportData = readResponse(doPost("/api/entities/export/" + entity.getId().getEntityType() + "/" + entity.getId()) + .andExpect(status().isOk()), new TypeReference() {}); + assertThat(exportData.getEntity()).isEqualTo(entity); + exportDataRequirements.accept(exportData); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + E importedEntity = (E) exportableEntitiesService.findEntityById(tenantId2, importResult.getSavedEntity().getId()); + assertThat(importedEntity).isNotNull(); + assertThat(importResult.getSavedEntity()).isEqualTo(importedEntity); + assertThat(importResult.getOldEntity()).isNull(); + + assertThat(importedEntity.getId()).isNotEqualTo(entity.getId()); + assertThat(importedEntity.getExternalId()).isEqualTo(entity.getId()); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + importedEntityRequirements.accept(importedEntity); + return importResult; + } + + private > EntityImportResult testExportImportBetweenTenants(E entity, Consumer importedEntityRequirements) throws Exception { + return testExportImportBetweenTenants(entity, exportData -> {}, importedEntityRequirements); + } + + + @SneakyThrows + private > List> importEntities(List> exportDataList) { + String requestJson = mapper.writerFor(new TypeReference>>() {}).writeValueAsString(exportDataList); + ResultActions resultActions = doPost("/api/entities/import", (Object) requestJson); + + try { + String responseJson = resultActions.andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); + JavaType type = mapper.getTypeFactory().constructCollectionType(List.class, + mapper.getTypeFactory().constructParametricType(EntityImportResult.class, exportDataList.get(0).getEntity().getClass())); + return mapper.readValue(responseJson, type); + } catch (AssertionError e) { + throw new AssertionError(readResponse(resultActions, String.class), e); + } + } + + private void logInAsTenantAdmin1() throws Exception { + login(tenantAdmin1.getEmail(), "12345678"); + } + + private void logInAsTenantAdmin2() throws Exception { + login(tenantAdmin2.getEmail(), "12345678"); + } + +} From 2a77c90d13871ac091e9840b1ba91557727e47ca Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 15:09:46 +0300 Subject: [PATCH 012/156] More tests for export/import api; refactoring --- .../DefaultEntitiesExportImportService.java | 29 +- .../exporting/ExportableEntitiesService.java | 6 +- .../sync/exporting/TenantEntitiesService.java | 27 ++ .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 8 +- ...EntitiesExportImportControllerSqlTest.java | 350 +++++++++++++----- .../server/dao/ExportableEntityDao.java | 8 +- .../dao/ExportableEntityRepository.java | 4 +- .../server/dao/TenantEntityDao.java | 11 +- .../server/dao/TenantEntityRepository.java | 26 ++ .../server/dao/asset/AssetDao.java | 2 +- .../server/dao/customer/CustomerDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 2 +- .../server/dao/device/DeviceDao.java | 2 +- .../server/dao/ota/OtaPackageDao.java | 2 +- .../server/dao/rule/RuleChainDao.java | 2 +- .../dao/sql/customer/CustomerRepository.java | 1 - .../dao/sql/device/JpaDeviceProfileDao.java | 5 + .../server/dao/sql/ota/JpaOtaPackageDao.java | 18 + .../dao/sql/ota/OtaPackageRepository.java | 3 +- .../dao/sql/rule/RuleChainRepository.java | 2 - .../server/dao/sql/user/JpaUserDao.java | 13 + .../server/dao/sql/user/UserRepository.java | 5 +- .../thingsboard/server/dao/user/UserDao.java | 2 +- 24 files changed, 411 insertions(+), 125 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index f399dbdbea..eeaef01f63 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -20,9 +20,12 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; @@ -37,7 +40,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; @Service @@ -46,7 +48,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final Map> daos = new HashMap<>(); + private final Map> daos = new HashMap<>(); protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -85,16 +87,22 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { - ExportableEntityDao dao = getDao(id.getEntityType()); + public & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id) { + TenantEntityDao dao = (TenantEntityDao) getDao(id.getEntityType()); return dao.findByTenantIdAndId(tenantId.getId(), id.getId()); } @Override public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { - ExportableEntityDao dao = getDao(externalId.getEntityType()); - return Optional.ofNullable(dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId())) - .orElseGet(() -> findEntityById(tenantId, externalId)); + EntityType entityType = externalId.getEntityType(); + if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { + ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); + E entity = dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + if (entity != null) { + return entity; + } + } + return findEntityById(tenantId, externalId); } @@ -114,15 +122,14 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return (EntityImportService) importServices.get(entityType); } - @SuppressWarnings("unchecked") - private ExportableEntityDao getDao(EntityType entityType) { - return (ExportableEntityDao) daos.get(entityType); + private TenantEntityDao getDao(EntityType entityType) { + return daos.get(entityType); } @Autowired private void setServices(Collection> exportServices, Collection> importServices, - Collection> daos) { + Collection> daos) { exportServices.forEach(entityExportService -> { this.exportServices.put(entityExportService.getEntityType(), entityExportService); }); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 2135d2aca1..1c1cf6bf7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -16,12 +16,12 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface ExportableEntitiesService { - - , I extends EntityId> E findEntityById(TenantId tenantId, I id); +public interface ExportableEntitiesService extends TenantEntitiesService { , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java new file mode 100644 index 0000000000..e3e25c21e8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting; + +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface TenantEntitiesService { + + & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 9e7cffd272..5525ba8e6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -27,7 +27,9 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; +import java.util.Collections; import java.util.HashSet; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -49,8 +51,8 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers().stream() - .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) + .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); Set newAssignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index eafe9c0fdb..2dcf4f7e7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -38,7 +38,13 @@ public class DeviceImportService extends BaseEntityImportService { - assertThat(importedAsset.getName()).isEqualTo(asset.getName()); - assertThat(importedAsset.getType()).isEqualTo(asset.getType()); - assertThat(importedAsset.getLabel()).isEqualTo(asset.getLabel()); - assertThat(importedAsset.getAdditionalInfo()).isEqualTo(asset.getAdditionalInfo()); - }); - - Customer customer = createCustomer(tenantId1, "Customer of tenant 1"); - testExportImportBetweenTenants(customer, importedCustomer -> { - assertThat(importedCustomer.getTitle()).isEqualTo(customer.getTitle()); - assertThat(importedCustomer.getCountry()).isEqualTo(customer.getCountry()); - assertThat(importedCustomer.getAddress()).isEqualTo(customer.getAddress()); - assertThat(importedCustomer.getEmail()).isEqualTo(customer.getEmail()); - }); + EntityExportData exportData = exportSingleEntity(asset.getId()); + assertThat(exportData.getEntity()).isEqualTo(asset); - DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); - DeviceProfileId importedDeviceProfileId = testExportImportBetweenTenants(deviceProfile, importedDeviceProfile -> { - assertThat(importedDeviceProfile.getName()).isEqualTo(deviceProfile.getName()); - assertThat(importedDeviceProfile.getType()).isEqualTo(deviceProfile.getType()); - assertThat(importedDeviceProfile.getTransportType()).isEqualTo(deviceProfile.getTransportType()); - assertThat(importedDeviceProfile.getProfileData()).isEqualTo(deviceProfile.getProfileData()); - assertThat(importedDeviceProfile.getDescription()).isEqualTo(deviceProfile.getDescription()); - }).getSavedEntity().getId(); + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, asset, tenantId2, importResult); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportSingleAsset_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); + EntityExportData exportData = exportSingleEntity(asset.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, asset, tenantId1, importResult); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_withCustomer_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + EntityExportData customerExportData = exportSingleEntity(customer.getId()); + EntityExportData assetExportData = exportSingleEntity(asset.getId()); + + logInAsTenantAdmin2(); + Customer importedCustomer = importEntities(List.of(customerExportData)).get(0).getSavedEntity(); + Asset importedAsset = importEntities(List.of(assetExportData)).get(0).getSavedEntity(); + + assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); + } + + @Test + public void testExportImportAsset_withCustomer_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + Asset importedAsset = this.importEntities(List.of(exportSingleEntity(asset.getId()))).get(0).getSavedEntity(); + + assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); + } + + private void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + + @Test + public void testExportImportSingleCustomer_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); + EntityExportData exportData = exportSingleEntity(customer.getId()); + assertThat(exportData.getEntity()).isEqualTo(customer); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, customer, tenantId2, importResult); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + @Test + public void testExportImportSingleCustomer_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); + EntityExportData exportData = exportSingleEntity(customer.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + + checkImportedEntity(tenantId1, customer, tenantId1, importResult); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + private void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + + + @Test + public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); - this.testExportImportBetweenTenants(device, deviceExportData -> { - assertThat(deviceExportData.getCredentials()).isEqualTo(credentials); - }, importedDevice -> { - assertThat(importedDevice.getName()).isEqualTo(device.getName()); - assertThat(importedDevice.getType()).isEqualTo(device.getType()); - assertThat(importedDevice.getDeviceData()).isEqualTo(device.getDeviceData()); - assertThat(importedDevice.getDeviceProfileId()).isEqualTo(importedDeviceProfileId); - assertThat(importedDevice.getLabel()).isEqualTo(device.getLabel()); - - DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); - assertThat(importedCredentials.getCredentialsId()).isEqualTo(credentials.getCredentialsId()); - assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); - assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); - }); + EntityExportData profileExportData = exportSingleEntity(deviceProfile.getId()); + assertThat(profileExportData.getEntity()).isEqualTo(deviceProfile); + + EntityExportData deviceExportData = exportSingleEntity(device.getId()); + assertThat(deviceExportData.getEntity()).isEqualTo(device); + assertThat(((DeviceExportData) deviceExportData).getCredentials()).isEqualTo(credentials); + DeviceCredentials exportedCredentials = ((DeviceExportData) deviceExportData).getCredentials(); + exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); + + logInAsTenantAdmin2(); + EntityImportResult profileImportResult = importEntities(List.of(profileExportData)).get(0); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult); + checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); + + + EntityImportResult deviceImportResult = importEntities(List.of(deviceExportData)).get(0); + Device importedDevice = deviceImportResult.getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult); + checkImportedDeviceData(device, importedDevice); + + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getId()).isNotEqualTo(credentials.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(exportedCredentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + } + + @Test + public void testExportImportDevice_sameTenant() throws Exception { + logInAsTenantAdmin1(); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile v1.0"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData deviceExportData = exportSingleEntity(device.getId()); + + EntityImportResult importResult = importEntities(List.of(deviceExportData)).get(0); + Device importedDevice = importResult.getSavedEntity(); + + checkImportedEntity(tenantId1, device, tenantId1, importResult); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); + assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + } + + private void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + + private void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + + @Test + public void testExportImportSingleRuleChain_betweenTenants() throws Exception { + logInAsTenantAdmin1(); RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); - this.testExportImportBetweenTenants(ruleChain, ruleChainExportData -> { - assertThat(ruleChainExportData.getMetaData()).isEqualTo(metaData); - }, importedRuleChain -> { - assertThat(importedRuleChain.getType()).isEqualTo(ruleChain.getType()); - assertThat(importedRuleChain.getName()).isEqualTo(ruleChain.getName()); - assertThat(importedRuleChain.isDebugMode()).isEqualTo(ruleChain.isDebugMode()); - assertThat(importedRuleChain.getConfiguration()).isEqualTo(ruleChain.getConfiguration()); - RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); - assertThat(importedMetaData.getConnections()).isEqualTo(metaData.getConnections()); - assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(metaData.getFirstNodeIndex()); - for (int i = 0; i < metaData.getNodes().size(); i++) { - RuleNode initialNode = metaData.getNodes().get(i); - RuleNode importedNode = importedMetaData.getNodes().get(i); - assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); - assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); - assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); - assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); - } - }); - - Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard of tenant 1"); - testExportImportBetweenTenants(dashboard, importedDashboard -> { - assertThat(importedDashboard.getTitle()).isEqualTo(dashboard.getTitle()); - assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - assertThat(importedDashboard.getImage()).isEqualTo(dashboard.getImage()); - assertThat(importedDashboard.isMobileHide()).isEqualTo(dashboard.isMobileHide()); - }); - } - - private , D extends EntityExportData> EntityImportResult testExportImportBetweenTenants(E entity, Consumer exportDataRequirements, Consumer importedEntityRequirements) throws Exception { + + EntityExportData exportData = exportSingleEntity(ruleChain.getId()); + assertThat(exportData.getEntity()).isEqualTo(ruleChain); + assertThat(((RuleChainExportData) exportData).getMetaData()).isEqualTo(metaData); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + @Test + public void testExportImportSingleRuleChain_sameTenant() throws Exception { logInAsTenantAdmin1(); - D exportData = readResponse(doPost("/api/entities/export/" + entity.getId().getEntityType() + "/" + entity.getId()) - .andExpect(status().isOk()), new TypeReference() {}); - assertThat(exportData.getEntity()).isEqualTo(entity); - exportDataRequirements.accept(exportData); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + + EntityExportData exportData = exportSingleEntity(ruleChain.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + private void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + + @Test + public void testExportImportSingleDashboard_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + assertThat(exportData.getEntity()).isEqualTo(dashboard); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } - E importedEntity = (E) exportableEntitiesService.findEntityById(tenantId2, importResult.getSavedEntity().getId()); - assertThat(importedEntity).isNotNull(); - assertThat(importResult.getSavedEntity()).isEqualTo(importedEntity); - assertThat(importResult.getOldEntity()).isNull(); + @Test + public void testExportImportSingleDashboard_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); - assertThat(importedEntity.getId()).isNotEqualTo(entity.getId()); - assertThat(importedEntity.getExternalId()).isEqualTo(entity.getId()); - assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); - importedEntityRequirements.accept(importedEntity); - return importResult; + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + + EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } - private > EntityImportResult testExportImportBetweenTenants(E entity, Consumer importedEntityRequirements) throws Exception { - return testExportImportBetweenTenants(entity, exportData -> {}, importedEntityRequirements); + private void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + + + private > void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { + E importedEntity = importResult.getSavedEntity(); + + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + assertThat(importResult.getOldEntity()).isEqualTo(initialEntity); + } } @@ -217,6 +391,12 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } } + private , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { + return readResponse(doPost("/api/entities/export/" + entityId.getEntityType() + "/" + entityId.getId().toString()) + .andExpect(status().isOk()), new TypeReference>() {}); + } + + private void logInAsTenantAdmin1() throws Exception { login(tenantAdmin1.getEmail(), "12345678"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 5b2b5a2366..9f2cd51952 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -15,16 +15,12 @@ */ package org.thingsboard.server.dao; -import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; import java.util.UUID; -public interface ExportableEntityDao { +public interface ExportableEntityDao> extends TenantEntityDao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - T findByTenantIdAndId(UUID tenantId, UUID id); - - EntityType getEntityType(); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java index b6a303a941..1ce43ffe12 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -17,10 +17,8 @@ package org.thingsboard.server.dao; import java.util.UUID; -public interface ExportableEntityRepository { +public interface ExportableEntityRepository extends TenantEntityRepository { D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - D findByTenantIdAndId(UUID tenantId, UUID id); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 17367c2cc7..53ffb7882a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,9 +15,18 @@ */ package org.thingsboard.server.dao; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.TenantId; -public interface TenantEntityDao { +import java.util.UUID; + +public interface TenantEntityDao { Long countByTenantId(TenantId tenantId); + + T findByTenantIdAndId(UUID tenantId, UUID id); + + EntityType getEntityType(); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java new file mode 100644 index 0000000000..8997f65ca8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import java.util.UUID; + +public interface TenantEntityRepository { + + D findByTenantIdAndId(UUID tenantId, UUID id); + + Long countByTenantId(UUID tenantId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 495886bcc4..c7cf6f38e9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -34,7 +34,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface AssetDao extends Dao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index 89672d2a73..df589711b9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -29,7 +29,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface CustomerDao extends Dao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 90353f49b3..ef6a739391 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DashboardDao extends Dao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 90f2b218a7..688e7246d4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -36,7 +36,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface DeviceDao extends Dao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index 445210ca3b..a24df5794c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,6 +21,6 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; -public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { +public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, TenantEntityDao { Long sumDataSizeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 59f287eadd..47f69cc99d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { +public interface RuleChainDao extends Dao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 1300fcdfa5..893f22503d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -38,5 +38,4 @@ public interface CustomerRepository extends JpaRepository, CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); - Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 539377a647..26e49e9c66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -116,6 +116,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao { +public interface OtaPackageRepository extends JpaRepository, TenantEntityRepository { @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index 888951778c..2b1bca32b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -63,8 +63,6 @@ public interface RuleChainRepository extends JpaRepository findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 6b2de9872a..42d331cab5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -18,6 +18,8 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -96,4 +98,15 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } + + @Override + public User findByTenantIdAndId(UUID tenantId, UUID id) { + return DaoUtil.getData(userRepository.findByTenantIdAndId(tenantId, id)); + } + + @Override + public EntityType getEntityType() { + return EntityType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 51f6138cc1..98b7cf0d00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -21,6 +21,8 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.TenantEntityDao; +import org.thingsboard.server.dao.TenantEntityRepository; import org.thingsboard.server.dao.model.sql.UserEntity; import java.util.UUID; @@ -28,7 +30,7 @@ import java.util.UUID; /** * @author Valerii Sosliuk */ -public interface UserRepository extends JpaRepository { +public interface UserRepository extends JpaRepository, TenantEntityRepository { UserEntity findByEmail(String email); @@ -47,5 +49,4 @@ public interface UserRepository extends JpaRepository { @Param("searchText") String searchText, Pageable pageable); - Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index b906b3278d..7e27c1bf00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object From 9b8fca99733b70e732f17085cd646c085a120703 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 19:22:16 +0300 Subject: [PATCH 013/156] Export/import api improvements --- .../server/controller/BaseController.java | 6 ++--- .../DefaultEntitiesExportImportService.java | 23 +++++++++------- .../exporting/ExportableEntitiesService.java | 5 ++-- .../sync/exporting/TenantEntitiesService.java | 27 ------------------- .../importing/impl/AssetImportService.java | 1 + .../impl/BaseEntityImportService.java | 1 - .../importing/impl/CustomerImportService.java | 1 + .../impl/DashboardImportService.java | 1 + .../importing/impl/DeviceImportService.java | 1 + .../impl/DeviceProfileImportService.java | 1 + .../impl/RuleChainImportService.java | 1 + ...EntitiesExportImportControllerSqlTest.java | 6 ++--- .../server/common/data/ExportableEntity.java | 6 +---- .../java/org/thingsboard/server/dao/Dao.java | 3 +++ .../server/dao/ExportableEntityDao.java | 2 +- .../dao/ExportableEntityRepository.java | 2 +- .../server/dao/TenantEntityDao.java | 11 +------- .../server/dao/TenantEntityRepository.java | 26 ------------------ .../server/dao/asset/AssetDao.java | 2 +- .../server/dao/customer/CustomerDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 2 +- .../server/dao/device/DeviceDao.java | 2 +- .../server/dao/ota/OtaPackageDao.java | 2 +- .../server/dao/rule/RuleChainDao.java | 4 +-- .../server/dao/sql/asset/JpaAssetDao.java | 5 ---- .../dao/sql/customer/CustomerRepository.java | 1 + .../dao/sql/customer/JpaCustomerDao.java | 5 ---- .../dao/sql/dashboard/JpaDashboardDao.java | 5 ---- .../server/dao/sql/device/JpaDeviceDao.java | 5 ---- .../dao/sql/device/JpaDeviceProfileDao.java | 10 ------- .../server/dao/sql/ota/JpaOtaPackageDao.java | 11 -------- .../dao/sql/ota/OtaPackageRepository.java | 3 +-- .../server/dao/sql/rule/JpaRuleChainDao.java | 5 ---- .../dao/sql/rule/RuleChainRepository.java | 2 ++ .../server/dao/sql/user/JpaUserDao.java | 13 --------- .../server/dao/sql/user/UserRepository.java | 5 ++-- .../thingsboard/server/dao/user/UserDao.java | 2 +- 37 files changed, 47 insertions(+), 163 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java delete mode 100644 dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 382929e1e7..1f683c7874 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -924,7 +924,7 @@ public abstract class BaseController { } } - protected & HasTenantId, I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { + protected , I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { boolean notifyEdgeService = false; EntityType entityType = savedEntity.getId().getEntityType(); @@ -954,7 +954,7 @@ public abstract class BaseController { case RULE_CHAIN: RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); if (RuleChainType.CORE.equals(ruleChainType)) { - tbClusterService.broadcastEntityStateChangeEvent(savedEntity.getTenantId(), savedEntity.getId(), + tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedEntity.getId(), isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); } if (RuleChainType.EDGE.equals(ruleChainType)) { @@ -981,7 +981,7 @@ public abstract class BaseController { log.error("Failed to log entity action", e); } if (notifyEdgeService) { - sendEntityNotificationMsg(savedEntity.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + sendEntityNotificationMsg(user.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index eeaef01f63..ef9da1f89d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -48,7 +49,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final Map> daos = new HashMap<>(); + private final Map> daos = new HashMap<>(); protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -86,12 +87,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - @Override - public & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id) { - TenantEntityDao dao = (TenantEntityDao) getDao(id.getEntityType()); - return dao.findByTenantIdAndId(tenantId.getId(), id.getId()); - } - @Override public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { EntityType entityType = externalId.getEntityType(); @@ -105,6 +100,12 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return findEntityById(tenantId, externalId); } + @Override + public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { + Dao dao = (Dao) getDao(id.getEntityType()); + return dao.findById(tenantId, id.getId()); + } + @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { @@ -122,14 +123,14 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return (EntityImportService) importServices.get(entityType); } - private TenantEntityDao getDao(EntityType entityType) { + private Dao getDao(EntityType entityType) { return daos.get(entityType); } @Autowired private void setServices(Collection> exportServices, Collection> importServices, - Collection> daos) { + Collection> daos) { exportServices.forEach(entityExportService -> { this.exportServices.put(entityExportService.getEntityType(), entityExportService); }); @@ -137,7 +138,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS this.importServices.put(entityImportService.getEntityType(), entityImportService); }); daos.forEach(dao -> { - this.daos.put(dao.getEntityType(), dao); + if (dao.getEntityType() != null) { + this.daos.put(dao.getEntityType(), dao); + } }); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 1c1cf6bf7f..9a47e98920 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -16,13 +16,14 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; -public interface ExportableEntitiesService extends TenantEntitiesService { +public interface ExportableEntitiesService { , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + , I extends EntityId> E findEntityById(TenantId tenantId, I id); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java deleted file mode 100644 index e3e25c21e8..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/TenantEntitiesService.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting; - -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; - -public interface TenantEntitiesService { - - & HasTenantId, I extends EntityId> E findEntityById(TenantId tenantId, I id); - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index b8328a43c3..b9013fd549 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -34,6 +34,7 @@ public class AssetImportService extends BaseEntityImportService assignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 2dcf4f7e7f..f35a20e312 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -34,6 +34,7 @@ public class DeviceImportService extends BaseEntityImportService { diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 464b5da401..f114af6625 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -28,11 +28,10 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; @@ -52,7 +51,6 @@ import org.thingsboard.server.service.sync.importing.EntityImportResult; import java.util.List; -import static org.assertj.core.api.Assertions.as; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -358,7 +356,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } - private > void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { + private & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { E importedEntity = importResult.getSavedEntity(); assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index f1e8c3754e..50569b7a02 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -17,9 +17,8 @@ package org.thingsboard.server.common.data; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; -public interface ExportableEntity extends HasId, HasTenantId, HasName { +public interface ExportableEntity extends HasId, HasName { I getId(); void setId(I id); @@ -27,7 +26,4 @@ public interface ExportableEntity extends HasId, HasTenan I getExternalId(); void setExternalId(I externalId); - TenantId getTenantId(); - void setTenantId(TenantId tenantId); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 932a6e5ec1..038be61d21 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import java.util.Collection; @@ -40,4 +41,6 @@ public interface Dao { void removeAllByIds(Collection ids); + default EntityType getEntityType() { return null; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 9f2cd51952..7e4d00286c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.ExportableEntity; import java.util.UUID; -public interface ExportableEntityDao> extends TenantEntityDao { +public interface ExportableEntityDao> extends Dao { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java index 1ce43ffe12..6e2d2e115a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -17,7 +17,7 @@ package org.thingsboard.server.dao; import java.util.UUID; -public interface ExportableEntityRepository extends TenantEntityRepository { +public interface ExportableEntityRepository { D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java index 53ffb7882a..17367c2cc7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityDao.java @@ -15,18 +15,9 @@ */ package org.thingsboard.server.dao; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.id.TenantId; -import java.util.UUID; - -public interface TenantEntityDao { +public interface TenantEntityDao { Long countByTenantId(TenantId tenantId); - - T findByTenantIdAndId(UUID tenantId, UUID id); - - EntityType getEntityType(); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java deleted file mode 100644 index 8997f65ca8..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/TenantEntityRepository.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao; - -import java.util.UUID; - -public interface TenantEntityRepository { - - D findByTenantIdAndId(UUID tenantId, UUID id); - - Long countByTenantId(UUID tenantId); - -} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index c7cf6f38e9..495886bcc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -34,7 +34,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, ExportableEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index df589711b9..89672d2a73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -29,7 +29,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, ExportableEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index ef6a739391..90353f49b3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, ExportableEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 688e7246d4..90f2b218a7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -36,7 +36,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, ExportableEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index a24df5794c..445210ca3b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -21,6 +21,6 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; -public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, TenantEntityDao { +public interface OtaPackageDao extends Dao, TenantEntityWithDataDao { Long sumDataSizeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index 47f69cc99d..1d6c1bf29a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -30,7 +30,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, ExportableEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. @@ -80,6 +80,4 @@ public interface RuleChainDao extends Dao, ExportableEntityDao findByTenantIdAndTypeAndName(TenantId tenantId, RuleChainType type, String name); - RuleChain findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index cd4fefac48..14788f8b38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -214,11 +214,6 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); } - @Override - public Asset findByTenantIdAndId(UUID tenantId, UUID id) { - return DaoUtil.getData(assetRepository.findByTenantIdAndId(tenantId, id)); - } - @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java index 893f22503d..1300fcdfa5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/CustomerRepository.java @@ -38,4 +38,5 @@ public interface CustomerRepository extends JpaRepository, CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); + Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 9cf74f1f33..17e1625a29 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -75,11 +75,6 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); } - @Override - public Device findByTenantIdAndId(UUID tenantId, UUID id) { - return findDeviceByTenantIdAndId(TenantId.fromUUID(tenantId), id); - } - @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 26e49e9c66..23e564b735 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -116,16 +116,6 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao, TenantEntityRepository { +public interface OtaPackageRepository extends JpaRepository { @Query(value = "SELECT COALESCE(SUM(ota.data_size), 0) FROM ota_package ota WHERE ota.tenant_id = :tenantId AND ota.data IS NOT NULL", nativeQuery = true) Long sumDataSizeByTenantId(@Param("tenantId") UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index e2d8aafef3..0d5fea6f98 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -114,11 +114,6 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java index 42d331cab5..6b2de9872a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/JpaUserDao.java @@ -18,8 +18,6 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -98,15 +96,4 @@ public class JpaUserDao extends JpaAbstractSearchTextDao imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } - - @Override - public User findByTenantIdAndId(UUID tenantId, UUID id) { - return DaoUtil.getData(userRepository.findByTenantIdAndId(tenantId, id)); - } - - @Override - public EntityType getEntityType() { - return EntityType.USER; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java index 98b7cf0d00..51f6138cc1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserRepository.java @@ -21,8 +21,6 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.security.Authority; -import org.thingsboard.server.dao.TenantEntityDao; -import org.thingsboard.server.dao.TenantEntityRepository; import org.thingsboard.server.dao.model.sql.UserEntity; import java.util.UUID; @@ -30,7 +28,7 @@ import java.util.UUID; /** * @author Valerii Sosliuk */ -public interface UserRepository extends JpaRepository, TenantEntityRepository { +public interface UserRepository extends JpaRepository { UserEntity findByEmail(String email); @@ -49,4 +47,5 @@ public interface UserRepository extends JpaRepository, TenantE @Param("searchText") String searchText, Pageable pageable); + Long countByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java index 7e27c1bf00..b906b3278d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserDao.java @@ -24,7 +24,7 @@ import org.thingsboard.server.dao.TenantEntityDao; import java.util.UUID; -public interface UserDao extends Dao, TenantEntityDao { +public interface UserDao extends Dao, TenantEntityDao { /** * Save or update user object From 18b2fd4664b2cb5a54b671421d577fd6e638d5a3 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 5 Apr 2022 15:01:26 +0300 Subject: [PATCH 014/156] Fix permission checks for export/import api --- .../EntitiesExportImportController.java | 94 +++++-------------- .../DefaultEntitiesExportImportService.java | 69 +++++++++----- .../sync/EntitiesExportImportService.java | 9 +- .../sync/exporting/EntityExportService.java | 5 +- .../exporting/ExportableEntitiesService.java | 14 ++- .../impl/BaseEntityExportService.java | 27 ++++-- .../sync/importing/EntityImportService.java | 5 +- .../importing/impl/AssetImportService.java | 8 +- .../impl/BaseEntityImportService.java | 92 ++++++++++++------ .../importing/impl/CustomerImportService.java | 6 +- .../impl/DashboardImportService.java | 10 +- .../importing/impl/DeviceImportService.java | 14 ++- .../impl/DeviceProfileImportService.java | 14 ++- .../impl/RuleChainImportService.java | 11 ++- .../server/dao/sql/alarm/JpaAlarmDao.java | 7 ++ .../server/dao/sql/edge/JpaEdgeDao.java | 5 + .../dao/sql/entityview/JpaEntityViewDao.java | 6 ++ .../dao/sql/resource/JpaTbResourceDao.java | 7 ++ .../server/dao/sql/rpc/JpaRpcDao.java | 7 ++ .../server/dao/sql/rule/JpaRuleNodeDao.java | 7 ++ .../server/dao/sql/tenant/JpaTenantDao.java | 6 ++ .../sql/usagerecord/JpaApiUsageStateDao.java | 7 ++ .../server/dao/sql/user/JpaUserDao.java | 7 ++ .../dao/sql/widget/JpaWidgetTypeDao.java | 7 ++ .../dao/sql/widget/JpaWidgetsBundleDao.java | 7 ++ 25 files changed, 291 insertions(+), 160 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index f158ff15e7..f4abd1b5f7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.EntitiesExportImportService; @@ -81,9 +80,10 @@ public class EntitiesExportImportController extends BaseController { public EntityExportData> exportSingleEntity(@PathVariable EntityType entityType, @PathVariable UUID id, @RequestParam Map exportSettingsParams) throws ThingsboardException { + SecurityUser user = getCurrentUser(); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); try { - return exportEntity(getTenantId(), entityId, toExportSettings(exportSettingsParams)); + return exportEntity(user, entityId, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -93,11 +93,12 @@ public class EntitiesExportImportController extends BaseController { public List>> exportEntitiesByIds(@PathVariable EntityType entityType, @RequestParam UUID[] ids, @RequestParam Map exportSettingsParams) throws ThingsboardException { + SecurityUser user = getCurrentUser(); List entitiesIds = Arrays.stream(ids) .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) .collect(Collectors.toList()); try { - return exportEntitiesByIds(getTenantId(), entitiesIds, toExportSettings(exportSettingsParams)); + return exportEntitiesByIds(user, entitiesIds, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -109,13 +110,13 @@ public class EntitiesExportImportController extends BaseController { @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "2147483647") int pageSize, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); entityTypeFilter.setEntityType(entityType); try { - return exportEntitiesByFilter(tenantId, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); + return exportEntitiesByFilter(user, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -127,10 +128,10 @@ public class EntitiesExportImportController extends BaseController { @RequestParam(defaultValue = "0") int page, @RequestParam(defaultValue = "2147483647") int pageSize, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); try { - return exportEntitiesByFilter(tenantId, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); + return exportEntitiesByFilter(user, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } @@ -141,12 +142,12 @@ public class EntitiesExportImportController extends BaseController { public List>> exportAllEntitiesByFilters(@RequestBody List filters, @RequestParam Map exportSettingsParams, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); try { List>> exportDataList = new ArrayList<>(); for (EntityFilter filter : filters) { - exportDataList.addAll(exportEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); + exportDataList.addAll(exportEntitiesByFilter(user, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); } return exportDataList; } catch (Exception e) { @@ -158,17 +159,17 @@ public class EntitiesExportImportController extends BaseController { public List>> exportEntitiesByQuery(@RequestBody EntityDataQuery entitiesQuery, @RequestParam Map exportSettingsParams, @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - TenantId tenantId = getTenantId(); + SecurityUser user = getCurrentUser(); CustomerId customerId = toCustomerId(customerUuid); try { - return exportEntitiesByQuery(tenantId, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); + return exportEntitiesByQuery(user, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); } catch (Exception e) { throw handleException(e); } } - private List>> exportEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { + private List>> exportEntitiesByFilter(SecurityUser user, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); pageLink.setPageSize(pageSize); @@ -176,43 +177,26 @@ public class EntitiesExportImportController extends BaseController { pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return exportEntitiesByQuery(tenantId, customerId, query, exportSettings); + return exportEntitiesByQuery(user, customerId, query, exportSettings); } - private List>> exportEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { - List entitiesIds = entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + private List>> exportEntitiesByQuery(SecurityUser user, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { + List entitiesIds = entityService.findEntityDataByQuery(user.getTenantId(), customerId, query).getData().stream() .map(EntityData::getEntityId) .collect(Collectors.toList()); - return exportEntitiesByIds(tenantId, entitiesIds, exportSettings); + return exportEntitiesByIds(user, entitiesIds, exportSettings); } - private List>> exportEntitiesByIds(TenantId tenantId, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { + private List>> exportEntitiesByIds(SecurityUser user, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { List>> exportDataList = new ArrayList<>(); for (EntityId entityId : entitiesIds) { - exportDataList.add(exportEntity(tenantId, entityId, exportSettings)); + exportDataList.add(exportEntity(user, entityId, exportSettings)); } return exportDataList; } - private , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - checkEntityId(entityId, Operation.READ); - - List relations = new LinkedList<>(); - if (exportSettings.isExportInboundRelations()) { - relations.addAll(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)); - } - if (exportSettings.isExportOutboundRelations()) { - relations.addAll(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)); - } - for (EntityRelation relation : relations) { - if (!relation.getFrom().equals(entityId)) { - checkEntityId(relation.getFrom(), Operation.READ); - } else if (!relation.getTo().equals(entityId)) { - checkEntityId(relation.getTo(), Operation.READ); - } - } - - return exportImportService.exportEntity(tenantId, entityId, exportSettings); + private , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + return exportImportService.exportEntity(user, entityId, exportSettings); } @@ -235,41 +219,7 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - for (EntityExportData> exportData : exportDataList) { - ExportableEntity existingEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), exportData.getEntity().getId()); - if (existingEntity != null) { - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.WRITE, existingEntity.getId(), existingEntity); - } else { - exportData.getEntity().setTenantId(user.getTenantId()); - accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE, null, exportData.getEntity()); - } - - List relations = new LinkedList<>(); - if (importSettings.isImportInboundRelations() && exportData.getInboundRelations() != null) { - relations.addAll(exportData.getInboundRelations()); - } - if (importSettings.isImportOutboundRelations() && exportData.getOutboundRelations() != null) { - relations.addAll(exportData.getOutboundRelations()); - } - for (EntityRelation relation : relations) { - EntityId otherEntityId = null; - if (!relation.getFrom().equals(exportData.getEntity().getId())) { - otherEntityId = relation.getFrom(); - } else if (!relation.getTo().equals(exportData.getEntity().getId())) { - otherEntityId = relation.getTo(); - } - if (otherEntityId != null) { - ExportableEntity otherEntity = exportableEntitiesService.findEntityByExternalId(user.getTenantId(), otherEntityId); - if (otherEntity != null) { - checkEntityId(otherEntity.getId(), Operation.WRITE); - } else { - throw new IllegalArgumentException("Relation is referencing non-existing entity"); - } - } - } - } - - return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); + return exportImportService.importEntities(user, exportDataList, importSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index ef9da1f89d..7d98ae9004 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -15,19 +15,23 @@ */ package org.thingsboard.server.service.sync; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; -import org.thingsboard.server.dao.TenantEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; @@ -36,7 +40,10 @@ import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -45,12 +52,15 @@ import java.util.stream.Collectors; @Service @TbCoreComponent +@RequiredArgsConstructor public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); + private final AccessControlService accessControlService; + protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD @@ -58,52 +68,67 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings) { + public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(tenantId, entityId, exportSettings); + return exportService.getExportData(user, entityId, exportSettings); } // TODO [viacheslav]: validate export data - @Transactional + @Transactional(rollbackFor = Exception.class) @Override - public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { + public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - return importService.importEntity(tenantId, exportData, importSettings); + return importService.importEntity(user, exportData, importSettings); } - @Transactional + @Transactional(rollbackFor = Exception.class) @Override - public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { - return exportDataList.stream() - .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) - // TODO [viacheslav]: order for rule chains (depending on references) - .map(exportData -> importEntity(tenantId, exportData, importSettings)) - .collect(Collectors.toList()); + public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + + List>> importResults = new ArrayList<>(); + for (EntityExportData> exportData : exportDataList) { + importResults.add(importEntity(user, exportData, importSettings)); + } + return importResults; } @Override - public , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId) { + public , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId) { EntityType entityType = externalId.getEntityType(); if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - E entity = dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); - if (entity != null) { - return entity; - } + return dao.findByTenantIdAndExternalId(user.getTenantId().getId(), externalId.getId()); } - return findEntityById(tenantId, externalId); + return findEntityById(user, externalId); } @Override - public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { + public , I extends EntityId> E findEntityById(SecurityUser user, I id) { Dao dao = (Dao) getDao(id.getEntityType()); - return dao.findById(tenantId, id.getId()); + return dao.findById(user.getTenantId(), id.getId()); + } + + + @Override + public void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException { + if (entity instanceof HasTenantId) { + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), (HasTenantId) entity); + } else if (entity != null) { + accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation); + } + } + + @Override + public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { + HasId entity = findEntityById(user, entityId); + checkPermission(user, entity, operation); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 0fbe75a866..447d514c68 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -16,8 +16,9 @@ package org.thingsboard.server.service.sync; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; @@ -27,10 +28,10 @@ import java.util.List; public interface EntitiesExportImportService { - , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); + , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); + , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException; - , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings); + List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index 6b57cbb557..681a398720 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { - D getExportData(TenantId tenantId, I entityId, EntityExportSettings exportSettings); + D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 9a47e98920..c46ea14b10 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -16,14 +16,22 @@ package org.thingsboard.server.service.sync.exporting; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; public interface ExportableEntitiesService { - , I extends EntityId> E findEntityByExternalId(TenantId tenantId, I externalId); + , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId); - , I extends EntityId> E findEntityById(TenantId tenantId, I id); + , I extends EntityId> E findEntityById(SecurityUser user, I id); + + + void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException; + + void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index a3a83196bb..66558634a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -18,14 +18,17 @@ package org.thingsboard.server.service.sync.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import java.util.List; @@ -38,19 +41,31 @@ public abstract class BaseEntityExportService inboundRelations = relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON); + List inboundRelations = relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + if (inboundRelations != null) { + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + } exportData.setInboundRelations(inboundRelations); } if (exportSettings.isExportOutboundRelations()) { - List outboundRelations = relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + if (outboundRelations != null) { + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); + } + } exportData.setOutboundRelations(outboundRelations); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java index 19947e8917..e3e40eea2a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java @@ -17,13 +17,14 @@ package org.thingsboard.server.service.sync.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { - EntityImportResult importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings); + EntityImportResult importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException; EntityType getEntityType(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index b9013fd549..326e15a91b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -33,9 +33,13 @@ public class AssetImportService extends BaseEntityImportService importEntity(TenantId tenantId, D exportData, EntityImportSettings importSettings) { + public EntityImportResult importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException { E entity = exportData.getEntity(); - E existingEntity = exportableEntitiesService.findEntityByExternalId(tenantId, entity.getId()); + E existingEntity = exportableEntitiesService.findEntityByExternalId(user, entity.getId()); entity.setExternalId(entity.getId()); + NewIdProvider idProvider = new NewIdProvider(user, entity, existingEntity, importSettings); + setOwner(user.getTenantId(), entity, idProvider); if (existingEntity == null) { entity.setId(null); + exportableEntitiesService.checkPermission(user, entity, Operation.CREATE); } else { entity.setId(existingEntity.getId()); + exportableEntitiesService.checkPermission(user, existingEntity, Operation.WRITE); } - E savedEntity = prepareAndSave(tenantId, entity, exportData, new NewIdProvider(entity, existingEntity, importSettings)); - importRelations(tenantId, savedEntity, existingEntity, exportData, importSettings); + E savedEntity = prepareAndSave(user.getTenantId(), entity, exportData, idProvider); + importRelations(user, savedEntity, existingEntity, exportData, importSettings); EntityImportResult importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -73,57 +80,71 @@ public abstract class BaseEntityImportService newRelations = new LinkedList<>(); if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { newRelations.addAll(exportData.getInboundRelations().stream() - .peek(relation -> { - relation.setTo(savedEntity.getId()); - relation.setFrom(getInternalId(tenantId, relation.getFrom())); - }) + .peek(relation -> relation.setTo(savedEntity.getId())) .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelations() && existingEntity != null) { - relationService.findByTo(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { - relationService.deleteRelation(tenantId, existingRelation); - }); + for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { newRelations.addAll(exportData.getOutboundRelations().stream() - .peek(relation -> { - relation.setTo(getInternalId(tenantId, relation.getTo())); - relation.setFrom(savedEntity.getId()); - }) + .peek(relation -> relation.setFrom(savedEntity.getId())) .collect(Collectors.toList())); + if (importSettings.isRemoveExistingRelations() && existingEntity != null) { - relationService.findByFrom(tenantId, savedEntity.getId(), RelationTypeGroup.COMMON).forEach(existingRelation -> { - relationService.deleteRelation(tenantId, existingRelation); - }); + for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } - newRelations.forEach(relation -> { - relationService.saveRelation(tenantId, relation); - }); + for (EntityRelation relation : newRelations) { + HasId otherEntity = null; + if (!relation.getTo().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user, relation.getTo()); + relation.setTo(otherEntity.getId()); + } + if (!relation.getFrom().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user, relation.getFrom()); + relation.setFrom(otherEntity.getId()); + } + if (otherEntity != null) { + exportableEntitiesService.checkPermission(user, otherEntity, Operation.WRITE); + } + + relationService.saveRelation(user.getTenantId(), relation); + } } - private ID getInternalId(TenantId tenantId, ID externalId) { + private , ID extends EntityId> IE findInternalEntity(SecurityUser user, ID externalId) { if (externalId == null || externalId.isNullUid()) { return null; } - HasId entity = exportableEntitiesService.findEntityByExternalId(tenantId, externalId); + IE entity = exportableEntitiesService.findEntityByExternalId(user, externalId); if (entity == null) { throw new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); } - return entity.getId(); + return entity; } @RequiredArgsConstructor protected class NewIdProvider { + private final SecurityUser user; private final E entity; private final E existingEntity; private final EntityImportSettings importSettings; @@ -132,26 +153,37 @@ public abstract class BaseEntityImportService ID get(TenantId tenantId, Function idExtractor) { + public ID get(Function idExtractor) { if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities() || ALWAYS_UPDATE_REFERENCED_IDS.contains(getEntityType())) { - return getInternalId(tenantId, idExtractor.apply(entity)); + return getInternalId(idExtractor.apply(this.entity)); } else { return idExtractor.apply(existingEntity); } } - public Set get(TenantId tenantId, Function> listExtractor, Function idGetter, BiConsumer idSetter) { + public Set get(Function> listExtractor, Function idGetter, BiConsumer idSetter) { if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() .peek(t -> { - idSetter.accept(t, getInternalId(tenantId, idGetter.apply(t))); + idSetter.accept(t, getInternalId(idGetter.apply(t))); }) .collect(Collectors.toSet()); } else { return listExtractor.apply(existingEntity); } } + + private ID getInternalId(ID externalId) { + try { + HasId entity = findInternalEntity(user, externalId); + exportableEntitiesService.checkPermission(user, entity, Operation.READ); + return entity.getId(); + } catch (ThingsboardException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index 92f7371d50..d17e6c8b41 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -33,8 +33,12 @@ public class CustomerImportService extends BaseEntityImportService assignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); + Set assignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); dashboard = dashboardService.saveDashboard(dashboard); for (ShortCustomerInfo customerInfo : assignedCustomers) { @@ -54,7 +58,7 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); - Set newAssignedCustomers = idProvider.get(tenantId, Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() + Set newAssignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); Set toUnassign = new HashSet<>(existingAssignedCustomers); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index f35a20e312..3b844f8657 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -33,12 +33,16 @@ public class DeviceImportService extends BaseEntityImportService { @@ -54,18 +58,19 @@ public class RuleChainImportService extends BaseEntityImportService { ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.get(tenantId, rc -> new RuleChainId(otherRuleChainUuid)).toString() + idProvider.get(rc -> new RuleChainId(otherRuleChainUuid)).toString() )); ruleNode.setConfiguration(ruleNodeConfig); }); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { - ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(tenantId, rc -> ruleChainConnectionInfo.getTargetRuleChainId())); + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(rc -> ruleChainConnectionInfo.getTargetRuleChainId())); }); ruleChain.setFirstRuleNodeId(null); if (ruleChain.getId() != null) { + // FIXME [viacheslav]: maybe no need to delete ruleChainService.deleteRuleNodes(tenantId, ruleChain.getId()); } ruleChain = ruleChainService.saveRuleChain(ruleChain); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index cdf04f47c5..8e165898a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -188,4 +189,10 @@ public class JpaAlarmDao extends JpaAbstractDao implements A log.trace("[{}] Try to delete entity alarm records using [{}]", tenantId, entityId); entityAlarmRepository.deleteByEntityId(entityId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 4be9c08849..802c9b99b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -193,4 +193,9 @@ public class JpaEdgeDao extends JpaAbstractSearchTextDao imple return list; } + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java index 8c515f13c0..52371706e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/JpaEntityViewDao.java @@ -200,4 +200,10 @@ public class JpaEntityViewDao extends JpaAbstractSearchTextDao implements RpcDao public Long deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + + @Override + public EntityType getEntityType() { + return EntityType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index a499e68da8..d49b54434f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.DaoUtil; @@ -50,4 +51,10 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search) { return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByTenantIdAndType(tenantId.getId(), type, search)); } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java index b985d45e8c..56a75f4d3b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/tenant/JpaTenantDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.tenant; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.id.TenantId; @@ -78,4 +79,9 @@ public class JpaTenantDao extends JpaAbstractSearchTextDao return DaoUtil.pageToPageData(tenantRepository.findTenantsIds(DaoUtil.toPageable(pageLink))).mapData(TenantId::fromUUID); } + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java index 7c6801042d..96a0ecaede 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/JpaApiUsageStateDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.usagerecord; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -68,4 +69,10 @@ public class JpaApiUsageStateDao extends JpaAbstractDao imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java index bbaf2529a7..eac0a15b1e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/JpaWidgetTypeDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.widget; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; @@ -73,4 +74,10 @@ public class JpaWidgetTypeDao extends JpaAbstractDao Date: Tue, 5 Apr 2022 17:12:51 +0300 Subject: [PATCH 015/156] Add option to find existing entity by name when importing; refactoring --- .../EntitiesExportImportController.java | 35 ++++++----- .../DefaultEntitiesExportImportService.java | 35 ++++++----- .../exporting/ExportableEntitiesService.java | 11 ++-- .../impl/BaseEntityExportService.java | 7 ++- .../sync/importing/EntityImportSettings.java | 1 + .../impl/BaseEntityImportService.java | 58 ++++++++++++------- .../server/dao/ExportableEntityDao.java | 2 + .../server/dao/asset/BaseAssetService.java | 2 + .../server/dao/sql/asset/JpaAssetDao.java | 5 ++ .../dao/sql/customer/JpaCustomerDao.java | 5 ++ .../sql/dashboard/DashboardRepository.java | 2 + .../dao/sql/dashboard/JpaDashboardDao.java | 5 ++ .../server/dao/sql/device/JpaDeviceDao.java | 5 ++ .../dao/sql/device/JpaDeviceProfileDao.java | 5 ++ .../server/dao/sql/rule/JpaRuleChainDao.java | 5 ++ .../dao/sql/rule/RuleChainRepository.java | 2 + 16 files changed, 127 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index f4abd1b5f7..25d48c54a7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -38,23 +37,18 @@ import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityTypeFilter; -import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -72,7 +66,6 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; - private final ExportableEntitiesService exportableEntitiesService; private final EntityService entityService; @@ -201,8 +194,8 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") - public List>> importEntity(@RequestBody List>> exportDataList, - @RequestParam Map importSettingsParams) throws ThingsboardException { + public List>> importEntities(@RequestBody List>> exportDataList, + @RequestParam Map importSettingsParams) throws ThingsboardException { SecurityUser user = getCurrentUser(); EntityImportSettings importSettings = toImportSettings(importSettingsParams); @@ -225,25 +218,31 @@ public class EntitiesExportImportController extends BaseController { private EntityExportSettings toExportSettings(Map exportSettingsParams) { return EntityExportSettings.builder() - .exportInboundRelations(getParam(exportSettingsParams, "exportInboundRelations", false, Boolean::parseBoolean)) - .exportOutboundRelations(getParam(exportSettingsParams, "exportOutboundRelations", false, Boolean::parseBoolean)) + .exportInboundRelations(getBooleanParam(exportSettingsParams, "exportInboundRelations", false)) + .exportOutboundRelations(getBooleanParam(exportSettingsParams, "exportOutboundRelations", false)) .build(); } private EntityImportSettings toImportSettings(Map importSettingsParams) { return EntityImportSettings.builder() - .importInboundRelations(getParam(importSettingsParams, "importInboundRelations", false, Boolean::parseBoolean)) - .importOutboundRelations(getParam(importSettingsParams, "importOutboundRelations", false, Boolean::parseBoolean)) - .removeExistingRelations(getParam(importSettingsParams, "removeExistingRelations", true, Boolean::parseBoolean)) - .updateReferencesToOtherEntities(getParam(importSettingsParams, "updateReferencesToOtherEntities", true, Boolean::parseBoolean)) + .findExistingByName(getBooleanParam(importSettingsParams, "findExistingByName", false)) + .importInboundRelations(getBooleanParam(importSettingsParams, "importInboundRelations", false)) + .importOutboundRelations(getBooleanParam(importSettingsParams, "importOutboundRelations", false)) + .removeExistingRelations(getBooleanParam(importSettingsParams, "removeExistingRelations", true)) + .updateReferencesToOtherEntities(getBooleanParam(importSettingsParams, "updateReferencesToOtherEntities", true)) .build(); } - protected T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { + + protected static boolean getBooleanParam(Map requestParams, String key, boolean defaultValue) { + return getParam(requestParams, key, defaultValue, Boolean::parseBoolean); + } + + protected static T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { return parsingFunction.apply(requestParams.getOrDefault(key, defaultValue.toString())); } - private CustomerId toCustomerId(UUID customerUuid) { + private static CustomerId toCustomerId(UUID customerUuid) { return new CustomerId(Optional.ofNullable(customerUuid).orElse(EntityId.NULL_UUID)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 7d98ae9004..8bfaa9fe80 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -41,14 +42,11 @@ import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.EntityImportSettings; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @Service @TbCoreComponent @@ -100,35 +98,46 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId) { + public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { EntityType entityType = externalId.getEntityType(); if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - return dao.findByTenantIdAndExternalId(user.getTenantId().getId(), externalId.getId()); + return dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } else { + return null; } - return findEntityById(user, externalId); } @Override - public , I extends EntityId> E findEntityById(SecurityUser user, I id) { + public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { Dao dao = (Dao) getDao(id.getEntityType()); - return dao.findById(user.getTenantId(), id.getId()); + E entity = dao.findById(tenantId, id.getId()); + if (entity instanceof HasTenantId && !((HasTenantId) entity).getTenantId().equals(tenantId)) { + return null; + } + return entity; + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { + ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); + return dao.findFirstByTenantIdAndName(tenantId.getId(), name); } @Override - public void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException { + public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { if (entity instanceof HasTenantId) { - accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation, entity.getId(), (HasTenantId) entity); + accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); } else if (entity != null) { - accessControlService.checkPermission(user, Resource.of(entity.getId().getEntityType()), operation); + accessControlService.checkPermission(user, Resource.of(entityType), operation); } } @Override public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { - HasId entity = findEntityById(user, entityId); - checkPermission(user, entity, operation); + HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); + checkPermission(user, entity, entityId.getEntityType(), operation); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index c46ea14b10..5cf31042e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -15,22 +15,25 @@ */ package org.thingsboard.server.service.sync.exporting; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; public interface ExportableEntitiesService { - , I extends EntityId> E findEntityByExternalId(SecurityUser user, I externalId); + , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId); - , I extends EntityId> E findEntityById(SecurityUser user, I id); + , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id); + , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); - void checkPermission(SecurityUser user, HasId entity, Operation operation) throws ThingsboardException; + + void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 66558634a4..f098485fd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -44,8 +44,11 @@ public abstract class BaseEntityExportService importEntity(SecurityUser user, D exportData, EntityImportSettings importSettings) throws ThingsboardException { E entity = exportData.getEntity(); - E existingEntity = exportableEntitiesService.findEntityByExternalId(user, entity.getId()); + E existingEntity = findExistingEntity(user.getTenantId(), entity, importSettings); entity.setExternalId(entity.getId()); @@ -65,10 +65,10 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) + .or(() -> { + if (importSettings.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); + } else { + return Optional.empty(); + } + }) + .orElse(null); + } + + private HasId findInternalEntity(TenantId tenantId, ID externalId) { + if (externalId == null || externalId.isNullUid()) return null; + + return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) + .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); + } + + private void importRelations(SecurityUser user, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) throws ThingsboardException { List newRelations = new LinkedList<>(); @@ -116,31 +138,21 @@ public abstract class BaseEntityImportService otherEntity = null; if (!relation.getTo().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user, relation.getTo()); + otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); relation.setTo(otherEntity.getId()); } if (!relation.getFrom().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user, relation.getFrom()); + otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); relation.setFrom(otherEntity.getId()); } if (otherEntity != null) { - exportableEntitiesService.checkPermission(user, otherEntity, Operation.WRITE); + exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); } relationService.saveRelation(user.getTenantId(), relation); } } - private , ID extends EntityId> IE findInternalEntity(SecurityUser user, ID externalId) { - if (externalId == null || externalId.isNullUid()) { - return null; - } - IE entity = exportableEntitiesService.findEntityByExternalId(user, externalId); - if (entity == null) { - throw new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId); - } - return entity; - } @RequiredArgsConstructor protected class NewIdProvider { @@ -175,12 +187,16 @@ public abstract class BaseEntityImportService ID getInternalId(ID externalId) { - try { - HasId entity = findInternalEntity(user, externalId); - exportableEntitiesService.checkPermission(user, entity, Operation.READ); + HasId entity = findInternalEntity(user.getTenantId(), externalId); + if (entity != null) { + try { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); + } catch (ThingsboardException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } return entity.getId(); - } catch (ThingsboardException e) { - throw new IllegalArgumentException(e.getMessage(), e); + } else { + return null; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 7e4d00286c..467389d9d0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -23,4 +23,6 @@ public interface ExportableEntityDao> extends Dao< T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + T findFirstByTenantIdAndName(UUID tenantId, String name); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index bf8d75d27a..4f31f3e8c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -25,6 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -110,6 +111,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ .orElse(null); } + @Transactional @CacheEvict(cacheNames = ASSET_CACHE, key = "{#asset.tenantId, #asset.name}") @Override public Asset saveAsset(Asset asset) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 14788f8b38..5988c2d2db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -214,6 +214,11 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); } + @Override + public Asset findFirstByTenantIdAndName(UUID tenantId, String name) { + return findAssetsByTenantIdAndName(tenantId, name).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 17e1625a29..fa09304427 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -75,6 +75,11 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); } + @Override + public Device findFirstByTenantIdAndName(UUID tenantId, String name) { + return findDeviceByTenantIdAndName(tenantId, name).orElse(null); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 23e564b735..b21a07442f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -116,6 +116,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); + RuleChainEntity findFirstByTenantIdAndName(UUID tenantId, String name); + } From cecf49a67dc60119ad9cd5b65fd49240d5ef5447 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 5 Apr 2022 18:08:26 +0300 Subject: [PATCH 016/156] Fix dashboard import to update entity aliases --- .../DefaultEntitiesExportImportService.java | 7 +++- .../impl/DashboardImportService.java | 33 ++++++++++++++++++- .../importing/impl/DeviceImportService.java | 2 +- .../server/dao/asset/BaseAssetService.java | 2 -- .../query/DefaultEntityQueryRepository.java | 2 +- 5 files changed, 40 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 8bfaa9fe80..90f6f1c778 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -74,13 +75,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - // TODO [viacheslav]: validate export data @Transactional(rollbackFor = Exception.class) @Override public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); + // TODO [viacheslav]: might throw DataIntegrityViolationException with cause of ConstraintViolationException: need to give normal error return importService.importEntity(user, exportData, importSettings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 7d0ff17ead..67f4ffc4a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -15,15 +15,22 @@ */ package org.thingsboard.server.service.sync.importing.impl; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; @@ -31,6 +38,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; @Service @@ -40,14 +49,36 @@ public class DashboardImportService extends BaseEntityImportService Optional.ofNullable(configuration.get("entityAliases"))) + .filter(JsonNode::isObject) + .ifPresent(entityAliases -> entityAliases.forEach(entityAlias -> { + Optional.ofNullable(entityAlias.get("filter")) + .filter(JsonNode::isObject) + .ifPresent(filter -> { + EntityFilter entityFilter = JacksonUtil.treeToValue(filter, EntityFilter.class); + EntityType entityType = DefaultEntityQueryRepository.resolveEntityType(entityFilter); + + String filterJson = filter.toString(); + String newFilterJson = UUID_PATTERN.matcher(filterJson).replaceAll(matchResult -> { + String uuid = matchResult.group(); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); + return idProvider.get(d -> entityId).toString(); + }); + ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); + }); + })); + + // TODO [viacheslav]: improve the code below if (dashboard.getId() == null) { Set assignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 3b844f8657..ccff72dcdb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -47,7 +47,7 @@ public class DeviceImportService extends BaseEntityImportService Date: Wed, 6 Apr 2022 00:29:22 +0300 Subject: [PATCH 017/156] Refactoring --- .../server/controller/AssetController.java | 6 +- .../server/controller/BaseController.java | 66 ------------------- .../server/controller/CustomerController.java | 10 ++- .../controller/DashboardController.java | 4 +- .../server/controller/DeviceController.java | 14 ++-- .../controller/DeviceProfileController.java | 25 ++++++- .../EntitiesExportImportController.java | 12 ++-- .../controller/RuleChainController.java | 6 +- .../service/action/EntityActionService.java | 59 ++++++++++++++++- .../impl/BaseEntityExportService.java | 15 +++-- .../sync/importing/EntityImportResult.java | 5 ++ .../importing/impl/AssetImportService.java | 9 +++ .../impl/BaseEntityImportService.java | 65 ++++++++++-------- .../importing/impl/CustomerImportService.java | 11 ++++ .../impl/DashboardImportService.java | 11 ++++ .../importing/impl/DeviceImportService.java | 13 ++++ .../impl/DeviceProfileImportService.java | 9 +++ .../impl/RuleChainImportService.java | 9 +++ .../server/utils/ThrowingRunnable.java | 24 +++++++ 19 files changed, 245 insertions(+), 128 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index b05e32efe7..5e44e1433a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -155,9 +155,7 @@ public class AssetController extends BaseController { checkEntity(asset.getId(), asset, Resource.ASSET); Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedAsset, null, asset.getId() == null); - + entityActionService.onAssetCreatedOrUpdated(getCurrentUser(), savedAsset, asset.getId() == null); return savedAsset; } catch (Exception e) { logEntityAction(emptyId(EntityType.ASSET), asset, @@ -667,7 +665,7 @@ public class AssetController extends BaseController { public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { - onEntityUpdatedOrCreated(user, importedAssetInfo.getEntity(), importedAssetInfo.getOldEntity(), !importedAssetInfo.isUpdated()); + entityActionService.onAssetCreatedOrUpdated(user, importedAssetInfo.getEntity(), !importedAssetInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 1f683c7874..f945cc30b7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; -import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.OtaPackage; @@ -69,7 +68,6 @@ import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityViewId; -import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.RpcId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -85,7 +83,6 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.plugin.ComponentDescriptor; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.rpc.Rpc; @@ -144,7 +141,6 @@ import javax.servlet.http.HttpServletResponse; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -923,66 +919,4 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } - - protected , I extends EntityId> void onEntityUpdatedOrCreated(User user, E savedEntity, E oldEntity, boolean isNewEntity) { - boolean notifyEdgeService = false; - - EntityType entityType = savedEntity.getId().getEntityType(); - switch (entityType) { - case DEVICE: - tbClusterService.onDeviceUpdated((Device) savedEntity, (Device) oldEntity); - break; - case DEVICE_PROFILE: - DeviceProfile deviceProfile = (DeviceProfile) savedEntity; - DeviceProfile oldDeviceProfile = (DeviceProfile) oldEntity; - boolean isFirmwareChanged = false; - boolean isSoftwareChanged = false; - if (!isNewEntity) { - if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { - isFirmwareChanged = true; - } - if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { - isSoftwareChanged = true; - } - } - tbClusterService.onDeviceProfileChange(deviceProfile, null); - tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), deviceProfile.getId(), - isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - otaPackageStateService.update(deviceProfile, isFirmwareChanged, isSoftwareChanged); - notifyEdgeService = true; - break; - case RULE_CHAIN: - RuleChainType ruleChainType = ((RuleChain) savedEntity).getType(); - if (RuleChainType.CORE.equals(ruleChainType)) { - tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedEntity.getId(), - isNewEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } - if (RuleChainType.EDGE.equals(ruleChainType)) { - if (!isNewEntity) { - notifyEdgeService = true; - } - } - break; - case ASSET: - case CUSTOMER: - case DASHBOARD: - if (!isNewEntity) { - notifyEdgeService = true; - } - break; - default: - throw new UnsupportedOperationException(); - } - - try { - logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : null, - isNewEntity ? ActionType.ADDED : ActionType.UPDATED, null); - } catch (ThingsboardException e) { - log.error("Failed to log entity action", e); - } - if (notifyEdgeService) { - sendEntityNotificationMsg(user.getTenantId(), savedEntity.getId(), isNewEntity ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 27037f36e7..b419121459 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -33,6 +33,7 @@ import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; @@ -150,7 +151,14 @@ public class CustomerController extends BaseController { checkEntity(customer.getId(), customer, Resource.CUSTOMER); Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - onEntityUpdatedOrCreated(getCurrentUser(), savedCustomer, null, customer.getId() == null); + + logEntityAction(savedCustomer.getId(), savedCustomer, + savedCustomer.getId(), + customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + + if (customer.getId() != null) { + sendEntityNotificationMsg(savedCustomer.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } return savedCustomer; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 4ea1ca83c7..b4b6d6be89 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -186,9 +186,7 @@ public class DashboardController extends BaseController { checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD); Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedDashboard, null, dashboard.getId() == null); - + entityActionService.onDashboardCreatedOrUpdated(getCurrentUser(), savedDashboard, dashboard.getId() == null); return savedDashboard; } catch (Exception e) { logEntityAction(emptyId(EntityType.DASHBOARD), dashboard, diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index feb58edb7a..2dff62aec6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,11 +74,11 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import javax.annotation.Nullable; import java.io.IOException; @@ -194,9 +194,7 @@ public class DeviceController extends BaseController { } Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, oldDevice, created); - + entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, oldDevice, created); return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -224,9 +222,7 @@ public class DeviceController extends BaseController { checkEntity(device.getId(), device, Resource.DEVICE); Device savedDevice = deviceService.saveDeviceWithCredentials(device, credentials); checkNotNull(savedDevice); - - onEntityUpdatedOrCreated(getCurrentUser(), savedDevice, device, device.getId() == null); - + entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, device, created); return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -1001,7 +997,7 @@ public class DeviceController extends BaseController { public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { - onEntityUpdatedOrCreated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); + entityActionService.onDeviceCreatedOrUpdated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java index 240e60be2c..25a35e77fa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java @@ -46,6 +46,7 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import java.util.List; +import java.util.Objects; import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_DATA; @@ -206,14 +207,32 @@ public class DeviceProfileController extends BaseController { checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE); - DeviceProfile oldDeviceProfile = null; + boolean isFirmwareChanged = false; + boolean isSoftwareChanged = false; + if (!created) { - oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); + DeviceProfile oldDeviceProfile = deviceProfileService.findDeviceProfileById(getTenantId(), deviceProfile.getId()); + if (!Objects.equals(deviceProfile.getFirmwareId(), oldDeviceProfile.getFirmwareId())) { + isFirmwareChanged = true; + } + if (!Objects.equals(deviceProfile.getSoftwareId(), oldDeviceProfile.getSoftwareId())) { + isSoftwareChanged = true; + } } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - onEntityUpdatedOrCreated(getCurrentUser(), deviceProfile, oldDeviceProfile, created); + tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); + tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), savedDeviceProfile.getId(), + created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + + logEntityAction(savedDeviceProfile.getId(), savedDeviceProfile, + null, + created ? ActionType.ADDED : ActionType.UPDATED, null); + + otaPackageStateService.update(savedDeviceProfile, isFirmwareChanged, isSoftwareChanged); + sendEntityNotificationMsg(getTenantId(), savedDeviceProfile.getId(), + deviceProfile.getId() == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); return savedDeviceProfile; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE_PROFILE), deviceProfile, diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 25d48c54a7..6d6a2e4619 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -200,11 +200,13 @@ public class EntitiesExportImportController extends BaseController { EntityImportSettings importSettings = toImportSettings(importSettingsParams); try { - return importEntities(user, exportDataList, importSettings) - .stream().peek(entityImportResult -> { - onEntityUpdatedOrCreated(user, entityImportResult.getSavedEntity(), entityImportResult.getOldEntity(), entityImportResult.getOldEntity() == null); - }) - .collect(Collectors.toList()); + List>> importResults = importEntities(user, exportDataList, importSettings); + for (EntityImportResult> entityImportResult : importResults) { + if (entityImportResult.getCallback() != null) { + entityImportResult.getCallback().run(); + } + } + return importResults; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index cb59951ac5..e27090c700 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -78,9 +78,11 @@ import org.thingsboard.server.service.security.permission.Resource; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -247,12 +249,10 @@ public class RuleChainController extends BaseController { try { boolean created = ruleChain.getId() == null; ruleChain.setTenantId(getCurrentUser().getTenantId()); - checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN); RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - - onEntityUpdatedOrCreated(getCurrentUser(), savedRuleChain, null, created); + entityActionService.onRuleChainCreatedOrUpdated(getCurrentUser(), savedRuleChain, created); return savedRuleChain; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index b8c0656bd2..d692e24bbe 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,12 +22,17 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -36,13 +41,20 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.service.security.model.SecurityUser; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -212,7 +224,7 @@ public class EntityActionService { } } - public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, + public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, ActionType actionType, Exception e, Object... additionalInfo) { if (customerId == null || customerId.isNullUid()) { customerId = user.getCustomerId(); @@ -267,4 +279,49 @@ public class EntityActionService { entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString()); } } + + + public void onDeviceCreatedOrUpdated(SecurityUser user, Device savedDevice, Device oldDevice, boolean newEntity) { + tbClusterService.onDeviceUpdated(savedDevice, oldDevice); + logEntityAction(user, savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + } + + public void onAssetCreatedOrUpdated(SecurityUser user, Asset savedAsset, boolean newEntity) { + logEntityAction(user, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (!newEntity) { + tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedAsset.getId(), + null, null, EdgeEventActionType.UPDATED); + } + } + + public void onDashboardCreatedOrUpdated(SecurityUser user, Dashboard savedDashboard, boolean newEntity) { + logEntityAction(user, savedDashboard.getId(), savedDashboard, null, + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (!newEntity) { + tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedDashboard.getId(), + null, null, EdgeEventActionType.UPDATED); + } + } + + public void onRuleChainCreatedOrUpdated(SecurityUser user, RuleChain savedRuleChain, boolean newEntity) { + if (RuleChainType.CORE.equals(savedRuleChain.getType())) { + tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + newEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + logEntityAction(user, savedRuleChain.getId(), savedRuleChain, null, + newEntity ? ActionType.ADDED : ActionType.UPDATED, null); + if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { + if (!newEntity) { + tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedRuleChain.getId(), + null, null, EdgeEventActionType.UPDATED); + } + } + } + + public void onCustomerCreatedOrUpdated(SecurityUser user, Customer savedCustomer) { + + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index f098485fd6..772e010b2b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -52,9 +52,16 @@ public abstract class BaseEntityExportService inboundRelations = relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); if (inboundRelations != null) { for (EntityRelation relation : inboundRelations) { exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); @@ -63,7 +70,7 @@ public abstract class BaseEntityExportService outboundRelations = relationService.findByFrom(user.getTenantId(), entityId, RelationTypeGroup.COMMON); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); if (outboundRelations != null) { for (EntityRelation relation : outboundRelations) { exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); @@ -71,12 +78,8 @@ public abstract class BaseEntityExportService> { private E savedEntity; private E oldEntity; + @JsonIgnore + private transient ThrowingRunnable callback; // to call when entity is successfully saved and transaction is committed } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 326e15a91b..1c031f50e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -23,7 +23,9 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.AssetExportData; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -43,6 +45,13 @@ public class AssetImportService extends BaseEntityImportService { + entityActionService.onAssetCreatedOrUpdated(user, savedAsset, oldAsset == null); + }; + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 42e9263e77..bf3c126c92 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; @@ -36,6 +37,7 @@ import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.LinkedList; @@ -52,6 +54,8 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); + importResult.setCallback(callback); return importResult; } @@ -84,30 +89,8 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) - .or(() -> { - if (importSettings.isFindExistingByName()) { - return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); - } else { - return Optional.empty(); - } - }) - .orElse(null); - } - - private HasId findInternalEntity(TenantId tenantId, ID externalId) { - if (externalId == null || externalId.isNullUid()) return null; - - return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) - .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) - .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); - } - - - private void importRelations(SecurityUser user, E savedEntity, E existingEntity, D exportData, EntityImportSettings importSettings) throws ThingsboardException { + protected ThrowingRunnable processAfterSavedAndGetCallback(SecurityUser user, E savedEntity, E oldEntity, D exportData, + EntityImportSettings importSettings, NewIdProvider idProvider) throws ThingsboardException { List newRelations = new LinkedList<>(); if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { @@ -115,7 +98,7 @@ public abstract class BaseEntityImportService relation.setTo(savedEntity.getId())) .collect(Collectors.toList())); - if (importSettings.isRemoveExistingRelations() && existingEntity != null) { + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); relationService.deleteRelation(user.getTenantId(), existingRelation); @@ -127,7 +110,7 @@ public abstract class BaseEntityImportService relation.setFrom(savedEntity.getId())) .collect(Collectors.toList())); - if (importSettings.isRemoveExistingRelations() && existingEntity != null) { + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); relationService.deleteRelation(user.getTenantId(), existingRelation); @@ -151,6 +134,34 @@ public abstract class BaseEntityImportService {}; + } + + + private E findExistingEntity(TenantId tenantId, E entity, EntityImportSettings importSettings) { + return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, entity.getId())) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) + .or(() -> { + if (importSettings.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(tenantId, getEntityType(), entity.getName())); + } else { + return Optional.empty(); + } + }) + .orElse(null); + } + + private HasId findInternalEntity(TenantId tenantId, ID externalId) { + if (externalId == null || externalId.isNullUid()) return null; + + return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) + .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index d17e6c8b41..336d9f68f2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -19,11 +19,15 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -42,6 +46,13 @@ public class CustomerImportService extends BaseEntityImportService { + entityActionService.onCustomerCreatedOrUpdated(user, savedCustomer); + }; + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 67f4ffc4a7..af24576d34 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -32,7 +33,10 @@ import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.HashSet; @@ -108,6 +112,13 @@ public class DashboardImportService extends BaseEntityImportService { + entityActionService.onDashboardCreatedOrUpdated(user, savedDashboard, oldDashboard == null); + }; + } + @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index ccff72dcdb..4212083dea 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -17,13 +17,18 @@ package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -31,6 +36,7 @@ import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; + private final TbClusterService clusterService; @Override protected void setOwner(TenantId tenantId, Device device, NewIdProvider idProvider) { @@ -52,6 +58,13 @@ public class DeviceImportService extends BaseEntityImportService { + clusterService.onDeviceUpdated(savedDevice, oldDevice); + }; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index b59d4fa23e..f9d8d2ddd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -23,7 +23,9 @@ import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; +import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -46,6 +48,13 @@ public class DeviceProfileImportService extends BaseEntityImportService { + + }; + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 8f0ff5dc7e..f6a96f24bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -27,7 +27,9 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.Optional; @@ -79,6 +81,13 @@ public class RuleChainImportService extends BaseEntityImportService { + + }; + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java new file mode 100644 index 0000000000..6965a79d19 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import org.thingsboard.server.common.data.exception.ThingsboardException; + +public interface ThrowingRunnable { + + void run() throws ThingsboardException; + +} From 12840c81e997ea0084292f608e9d7b522e786ece Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 6 Apr 2022 21:39:26 +0300 Subject: [PATCH 018/156] Refactor events sending on import; export api with ExportRequest of different types --- .../server/controller/AssetController.java | 20 +- .../controller/DashboardController.java | 10 +- .../server/controller/DeviceController.java | 28 ++- .../EntitiesExportImportController.java | 199 ++++++------------ .../controller/RuleChainController.java | 17 +- .../service/action/EntityActionService.java | 61 +----- .../DefaultEntitiesExportImportService.java | 2 +- .../sync/EntitiesExportImportService.java | 2 +- .../sync/exporting/EntityExportService.java | 1 + .../request}/EntityExportSettings.java | 8 +- .../request/EntityFilterExportRequest.java | 37 ++++ .../data/request/EntityListExportRequest.java | 35 +++ .../request/EntityQueryExportRequest.java | 35 +++ .../data/request/EntityTypeExportRequest.java | 37 ++++ .../exporting/data/request/ExportRequest.java | 41 ++++ .../data/request/ExportRequestType.java | 25 +++ .../request/SingleEntityExportRequest.java | 33 +++ .../impl/BaseEntityExportService.java | 2 +- .../importing/data/request/ImportRequest.java | 32 +++ .../importing/impl/AssetImportService.java | 9 +- .../impl/BaseEntityImportService.java | 10 +- .../importing/impl/CustomerImportService.java | 9 +- .../impl/DashboardImportService.java | 12 +- .../importing/impl/DeviceImportService.java | 8 +- .../impl/DeviceProfileImportService.java | 19 +- .../impl/RuleChainImportService.java | 14 +- .../server/utils/ThrowingRunnable.java | 7 + ...EntitiesExportImportControllerSqlTest.java | 2 + 28 files changed, 479 insertions(+), 236 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/exporting/{ => data/request}/EntityExportSettings.java (79%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 5e44e1433a..e48f2dcf23 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -155,7 +155,9 @@ public class AssetController extends BaseController { checkEntity(asset.getId(), asset, Resource.ASSET); Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - entityActionService.onAssetCreatedOrUpdated(getCurrentUser(), savedAsset, asset.getId() == null); + + onAssetCreatedOrUpdated(savedAsset, asset.getId() != null, getCurrentUser()); + return savedAsset; } catch (Exception e) { logEntityAction(emptyId(EntityType.ASSET), asset, @@ -164,6 +166,20 @@ public class AssetController extends BaseController { } } + private void onAssetCreatedOrUpdated(Asset asset, boolean updated, SecurityUser user) { + try { + logEntityAction(user, asset.getId(), asset, + asset.getCustomerId(), + updated ? ActionType.UPDATED : ActionType.ADDED, null); + } catch (ThingsboardException e) { + log.error("Failed to log entity action", e); + } + + if (updated) { + sendEntityNotificationMsg(asset.getTenantId(), asset.getId(), EdgeEventActionType.UPDATED); + } + } + @ApiOperation(value = "Delete asset (deleteAsset)", notes = "Deletes the asset and all the relations (from and to the asset). Referencing non-existing asset Id will cause an error." + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -665,7 +681,7 @@ public class AssetController extends BaseController { public BulkImportResult processAssetsBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return assetBulkImportService.processBulkImport(request, user, importedAssetInfo -> { - entityActionService.onAssetCreatedOrUpdated(user, importedAssetInfo.getEntity(), !importedAssetInfo.isUpdated()); + onAssetCreatedOrUpdated(importedAssetInfo.getEntity(), importedAssetInfo.isUpdated(), user); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index b4b6d6be89..49c33045a5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -186,7 +186,15 @@ public class DashboardController extends BaseController { checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD); Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - entityActionService.onDashboardCreatedOrUpdated(getCurrentUser(), savedDashboard, dashboard.getId() == null); + + logEntityAction(savedDashboard.getId(), savedDashboard, + null, + dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + + if (dashboard.getId() != null) { + sendEntityNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } + return savedDashboard; } catch (Exception e) { logEntityAction(emptyId(EntityType.DASHBOARD), dashboard, diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 2dff62aec6..03f0996ea9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,11 +74,11 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; +import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; import javax.annotation.Nullable; import java.io.IOException; @@ -194,7 +194,9 @@ public class DeviceController extends BaseController { } Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, oldDevice, created); + + onDeviceCreatedOrUpdated(savedDevice, oldDevice, !created, getCurrentUser()); + return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -222,7 +224,11 @@ public class DeviceController extends BaseController { checkEntity(device.getId(), device, Resource.DEVICE); Device savedDevice = deviceService.saveDeviceWithCredentials(device, credentials); checkNotNull(savedDevice); - entityActionService.onDeviceCreatedOrUpdated(getCurrentUser(), savedDevice, device, created); + tbClusterService.onDeviceUpdated(savedDevice, device); + logEntityAction(savedDevice.getId(), savedDevice, + savedDevice.getCustomerId(), + device.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null); + return savedDevice; } catch (Exception e) { logEntityAction(emptyId(EntityType.DEVICE), device, @@ -231,6 +237,18 @@ public class DeviceController extends BaseController { } } + private void onDeviceCreatedOrUpdated(Device savedDevice, Device oldDevice, boolean updated, SecurityUser user) { + tbClusterService.onDeviceUpdated(savedDevice, oldDevice); + + try { + logEntityAction(user, savedDevice.getId(), savedDevice, + savedDevice.getCustomerId(), + updated ? ActionType.UPDATED : ActionType.ADDED, null); + } catch (ThingsboardException e) { + log.error("Failed to log entity action", e); + } + } + @ApiOperation(value = "Delete device (deleteDevice)", notes = "Deletes the device, it's credentials and all the relations (from and to the device). Referencing non-existing device Id will cause an error." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -997,7 +1015,7 @@ public class DeviceController extends BaseController { public BulkImportResult processDevicesBulkImport(@RequestBody BulkImportRequest request) throws Exception { SecurityUser user = getCurrentUser(); return deviceBulkImportService.processBulkImport(request, user, importedDeviceInfo -> { - entityActionService.onDeviceCreatedOrUpdated(user, importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), !importedDeviceInfo.isUpdated()); + onDeviceCreatedOrUpdated(importedDeviceInfo.getEntity(), importedDeviceInfo.getOldEntity(), importedDeviceInfo.isUpdated(), user); }); } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 6d6a2e4619..366eaec41a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,18 +17,16 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -41,13 +39,17 @@ import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityQueryExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -60,7 +62,6 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @RestController @RequestMapping("/api/entities") -@PreAuthorize("hasAuthority('TENANT_ADMIN')") @TbCoreComponent @RequiredArgsConstructor public class EntitiesExportImportController extends BaseController { @@ -68,101 +69,79 @@ public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; private final EntityService entityService; - - @PostMapping("/export/{entityType}/{id}") - public EntityExportData> exportSingleEntity(@PathVariable EntityType entityType, - @PathVariable UUID id, - @RequestParam Map exportSettingsParams) throws ThingsboardException { + @PostMapping("/export") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); try { - return exportEntity(user, entityId, toExportSettings(exportSettingsParams)); + return exportEntitiesByRequest(user, exportRequest); } catch (Exception e) { throw handleException(e); } } - @PostMapping(value = "/export/{entityType}", params = "ids") - public List>> exportEntitiesByIds(@PathVariable EntityType entityType, - @RequestParam UUID[] ids, - @RequestParam Map exportSettingsParams) throws ThingsboardException { + @PostMapping(value = "/export", params = {"multiple"}) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { SecurityUser user = getCurrentUser(); - List entitiesIds = Arrays.stream(ids) - .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) - .collect(Collectors.toList()); try { - return exportEntitiesByIds(user, entitiesIds, toExportSettings(exportSettingsParams)); + List>> exportDataList = new ArrayList<>(); + for (ExportRequest exportRequest : exportRequests) { + exportDataList.addAll(exportEntitiesByRequest(user, exportRequest)); + } + return exportDataList; } catch (Exception e) { throw handleException(e); } } - @PostMapping(value = "/export/{entityType}") - public List>> exportEntitiesByEntityType(@PathVariable EntityType entityType, - @RequestParam Map exportSettingsParams, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "2147483647") int pageSize, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - try { - return exportEntitiesByFilter(user, customerId, entityTypeFilter, page, pageSize, toExportSettings(exportSettingsParams)); - } catch (Exception e) { - throw handleException(e); - } - } + private List>> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { + List entitiesIds = findEntitiesForRequest(user, request); - @PostMapping("/exportByFilter") - public List>> exportEntitiesByFilter(@RequestBody EntityFilter filter, - @RequestParam Map exportSettingsParams, - @RequestParam(defaultValue = "0") int page, - @RequestParam(defaultValue = "2147483647") int pageSize, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - try { - return exportEntitiesByFilter(user, customerId, filter, page, pageSize, toExportSettings(exportSettingsParams)); - } catch (Exception e) { - throw handleException(e); + List>> exportDataList = new ArrayList<>(); + for (EntityId entityId : entitiesIds) { + exportDataList.add(exportImportService.exportEntity(user, entityId, request.getExportSettings())); } + return exportDataList; } - // FIXME: too aggressive - @PostMapping("/exportByFilters") - public List>> exportAllEntitiesByFilters(@RequestBody List filters, - @RequestParam Map exportSettingsParams, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - try { - List>> exportDataList = new ArrayList<>(); - for (EntityFilter filter : filters) { - exportDataList.addAll(exportEntitiesByFilter(user, customerId, filter, 0, Integer.MAX_VALUE, toExportSettings(exportSettingsParams))); + private List findEntitiesForRequest(SecurityUser user, ExportRequest request) { + switch (request.getType()) { + case SINGLE_ENTITY: { + return List.of(((SingleEntityExportRequest) request).getEntityId()); } - return exportDataList; - } catch (Exception e) { - throw handleException(e); - } - } + case ENTITY_LIST: { + return ((EntityListExportRequest) request).getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(exportRequest.getEntityType()); - @PostMapping("/exportByQuery") - public List>> exportEntitiesByQuery(@RequestBody EntityDataQuery entitiesQuery, - @RequestParam Map exportSettingsParams, - @RequestParam(name = "customerId", required = false) UUID customerUuid) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - CustomerId customerId = toCustomerId(customerUuid); - try { - return exportEntitiesByQuery(user, customerId, entitiesQuery, toExportSettings(exportSettingsParams)); - } catch (Exception e) { - throw handleException(e); + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); + return findEntitiesByFilter(user.getTenantId(), customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case ENTITY_FILTER: { + EntityFilterExportRequest exportRequest = (EntityFilterExportRequest) request; + EntityFilter filter = exportRequest.getFilter(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); + return findEntitiesByFilter(user.getTenantId(), customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case ENTITY_QUERY:{ + EntityQueryExportRequest exportRequest = (EntityQueryExportRequest) request; + EntityDataQuery query = exportRequest.getQuery(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); + return findEntitiesByQuery(user.getTenantId(), customerId, query); + } + default: + throw new IllegalArgumentException("Export request is not supported"); } } - - private List>> exportEntitiesByFilter(SecurityUser user, CustomerId customerId, EntityFilter filter, int page, int pageSize, EntityExportSettings exportSettings) throws ThingsboardException { + private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); pageLink.setPageSize(pageSize); @@ -170,37 +149,23 @@ public class EntitiesExportImportController extends BaseController { pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return exportEntitiesByQuery(user, customerId, query, exportSettings); + return findEntitiesByQuery(tenantId, customerId, query); } - private List>> exportEntitiesByQuery(SecurityUser user, CustomerId customerId, EntityDataQuery query, EntityExportSettings exportSettings) throws ThingsboardException { - List entitiesIds = entityService.findEntityDataByQuery(user.getTenantId(), customerId, query).getData().stream() + private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { + return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() .map(EntityData::getEntityId) .collect(Collectors.toList()); - return exportEntitiesByIds(user, entitiesIds, exportSettings); - } - - private List>> exportEntitiesByIds(SecurityUser user, List entitiesIds, EntityExportSettings exportSettings) throws ThingsboardException { - List>> exportDataList = new ArrayList<>(); - for (EntityId entityId : entitiesIds) { - exportDataList.add(exportEntity(user, entityId, exportSettings)); - } - return exportDataList; - } - - private , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - return exportImportService.exportEntity(user, entityId, exportSettings); } @PostMapping("/import") - public List>> importEntities(@RequestBody List>> exportDataList, - @RequestParam Map importSettingsParams) throws ThingsboardException { + public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); - EntityImportSettings importSettings = toImportSettings(importSettingsParams); - try { - List>> importResults = importEntities(user, exportDataList, importSettings); + List>> importResults = exportImportService.importEntities(user, + importRequest.getExportDataList(), importRequest.getImportSettings()); + for (EntityImportResult> entityImportResult : importResults) { if (entityImportResult.getCallback() != null) { entityImportResult.getCallback().run(); @@ -212,40 +177,4 @@ public class EntitiesExportImportController extends BaseController { } } - - public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - return exportImportService.importEntities(user, exportDataList, importSettings); - } - - - private EntityExportSettings toExportSettings(Map exportSettingsParams) { - return EntityExportSettings.builder() - .exportInboundRelations(getBooleanParam(exportSettingsParams, "exportInboundRelations", false)) - .exportOutboundRelations(getBooleanParam(exportSettingsParams, "exportOutboundRelations", false)) - .build(); - } - - private EntityImportSettings toImportSettings(Map importSettingsParams) { - return EntityImportSettings.builder() - .findExistingByName(getBooleanParam(importSettingsParams, "findExistingByName", false)) - .importInboundRelations(getBooleanParam(importSettingsParams, "importInboundRelations", false)) - .importOutboundRelations(getBooleanParam(importSettingsParams, "importOutboundRelations", false)) - .removeExistingRelations(getBooleanParam(importSettingsParams, "removeExistingRelations", true)) - .updateReferencesToOtherEntities(getBooleanParam(importSettingsParams, "updateReferencesToOtherEntities", true)) - .build(); - } - - - protected static boolean getBooleanParam(Map requestParams, String key, boolean defaultValue) { - return getParam(requestParams, key, defaultValue, Boolean::parseBoolean); - } - - protected static T getParam(Map requestParams, String key, T defaultValue, Function parsingFunction) { - return parsingFunction.apply(requestParams.getOrDefault(key, defaultValue.toString())); - } - - private static CustomerId toCustomerId(UUID customerUuid) { - return new CustomerId(Optional.ofNullable(customerUuid).orElse(EntityId.NULL_UUID)); - } - } diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index e27090c700..6479b838b2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -249,10 +249,25 @@ public class RuleChainController extends BaseController { try { boolean created = ruleChain.getId() == null; ruleChain.setTenantId(getCurrentUser().getTenantId()); + checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN); RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); - entityActionService.onRuleChainCreatedOrUpdated(getCurrentUser(), savedRuleChain, created); + + if (RuleChainType.CORE.equals(savedRuleChain.getType())) { + tbClusterService.broadcastEntityStateChangeEvent(ruleChain.getTenantId(), savedRuleChain.getId(), + created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } + + logEntityAction(savedRuleChain.getId(), savedRuleChain, + null, + created ? ActionType.ADDED : ActionType.UPDATED, null); + + if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { + if (!created) { + sendEntityNotificationMsg(savedRuleChain.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } + } return savedRuleChain; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index d692e24bbe..2a64db7385 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,18 +22,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -41,20 +37,12 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; -import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleChainType; -import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.service.security.model.SecurityUser; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -235,6 +223,9 @@ public class EntityActionService { auditLogService.logEntityAction(user.getTenantId(), customerId, user.getId(), user.getName(), entityId, entity, actionType, e, additionalInfo); } + public void sendEntityNotificationMsgToEdgeService(TenantId tenantId, EntityId entityId, EdgeEventActionType action) { + tbClusterService.sendNotificationMsgToEdgeService(tenantId, null, entityId, null, null, action); + } private T extractParameter(Class clazz, int index, Object... additionalInfo) { T result = null; @@ -280,48 +271,4 @@ public class EntityActionService { } } - - public void onDeviceCreatedOrUpdated(SecurityUser user, Device savedDevice, Device oldDevice, boolean newEntity) { - tbClusterService.onDeviceUpdated(savedDevice, oldDevice); - logEntityAction(user, savedDevice.getId(), savedDevice, savedDevice.getCustomerId(), - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - } - - public void onAssetCreatedOrUpdated(SecurityUser user, Asset savedAsset, boolean newEntity) { - logEntityAction(user, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (!newEntity) { - tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedAsset.getId(), - null, null, EdgeEventActionType.UPDATED); - } - } - - public void onDashboardCreatedOrUpdated(SecurityUser user, Dashboard savedDashboard, boolean newEntity) { - logEntityAction(user, savedDashboard.getId(), savedDashboard, null, - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (!newEntity) { - tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedDashboard.getId(), - null, null, EdgeEventActionType.UPDATED); - } - } - - public void onRuleChainCreatedOrUpdated(SecurityUser user, RuleChain savedRuleChain, boolean newEntity) { - if (RuleChainType.CORE.equals(savedRuleChain.getType())) { - tbClusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), - newEntity ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } - logEntityAction(user, savedRuleChain.getId(), savedRuleChain, null, - newEntity ? ActionType.ADDED : ActionType.UPDATED, null); - if (RuleChainType.EDGE.equals(savedRuleChain.getType())) { - if (!newEntity) { - tbClusterService.sendNotificationMsgToEdgeService(user.getTenantId(), null, savedRuleChain.getId(), - null, null, EdgeEventActionType.UPDATED); - } - } - } - - public void onCustomerCreatedOrUpdated(SecurityUser user, Customer savedCustomer) { - - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 90f6f1c778..9bca5f8bc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 447d514c68..3fa9b13406 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -19,7 +19,7 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportSettings; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index 681a398720..c09cbcd36c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java index 36552b2c12..01e4f8d068 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java @@ -13,17 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exporting.data.request; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; @Data -@AllArgsConstructor -@NoArgsConstructor -@Builder public class EntityExportSettings { private boolean exportInboundRelations; private boolean exportOutboundRelations; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java new file mode 100644 index 0000000000..00ef0604e5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.EntityFilter; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityFilterExportRequest extends ExportRequest { + + private EntityFilter filter; + private int page; + private int pageSize; + private CustomerId customerId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_FILTER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java new file mode 100644 index 0000000000..01309a421b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityListExportRequest extends ExportRequest { + + private List entitiesIds; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_LIST; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java new file mode 100644 index 0000000000..85232821d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityQueryExportRequest extends ExportRequest { + + private EntityDataQuery query; + private CustomerId customerId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_QUERY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java new file mode 100644 index 0000000000..82cb023258 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EntityTypeExportRequest extends ExportRequest { + + private EntityType entityType; + private int page; + private int pageSize; + private CustomerId customerId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java new file mode 100644 index 0000000000..c140786d06 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; +import lombok.Data; + +@JsonTypeInfo(use = Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), + @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), + @Type(name = "ENTITY_FILTER", value = EntityFilterExportRequest.class), + @Type(name = "ENTITY_QUERY", value = EntityQueryExportRequest.class) +}) +@Data +public abstract class ExportRequest { + + private EntityExportSettings exportSettings; + + @JsonIgnore + public abstract ExportRequestType getType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java new file mode 100644 index 0000000000..07e5c946ba --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +public enum ExportRequestType { + SINGLE_ENTITY, + ENTITY_LIST, + ENTITY_TYPE, + + ENTITY_FILTER, + ENTITY_QUERY +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java new file mode 100644 index 0000000000..fe07ba7534 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.data.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@EqualsAndHashCode(callSuper = true) +@Data +public class SingleEntityExportRequest extends ExportRequest { + + private EntityId entityId; + + @Override + public ExportRequestType getType() { + return ExportRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 772e010b2b..5d693e25b6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java new file mode 100644 index 0000000000..cc2eeba618 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.importing.data.request; + +import lombok.Data; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; + +import java.util.List; + +@Data +public class ImportRequest { + + private List>> exportDataList; + private EntityImportSettings importSettings; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 1c031f50e4..9a79c26515 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; @@ -47,9 +48,11 @@ public class AssetImportService extends BaseEntityImportService { - entityActionService.onAssetCreatedOrUpdated(user, savedAsset, oldAsset == null); - }; + return super.getCallback(user, savedAsset, oldAsset).andThen(() -> { + if (oldAsset != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index bf3c126c92..9715fe008e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -20,8 +20,11 @@ import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasCustomerId; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -56,6 +59,8 @@ public abstract class BaseEntityImportService {}; + return () -> { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); + }; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index 336d9f68f2..a03ee60569 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -48,9 +49,11 @@ public class CustomerImportService extends BaseEntityImportService { - entityActionService.onCustomerCreatedOrUpdated(user, savedCustomer); - }; + return super.getCallback(user, savedCustomer, oldCustomer).andThen(() -> { + if (oldCustomer != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index af24576d34..e80e699c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -23,7 +23,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; -import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -35,14 +35,12 @@ import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -114,9 +112,11 @@ public class DashboardImportService extends BaseEntityImportService { - entityActionService.onDashboardCreatedOrUpdated(user, savedDashboard, oldDashboard == null); - }; + return super.getCallback(user, savedDashboard, oldDashboard).andThen(() -> { + if (oldDashboard != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 4212083dea..7484fb98ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -17,17 +17,14 @@ package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; -import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; @Service @@ -36,7 +33,6 @@ import org.thingsboard.server.utils.ThrowingRunnable; public class DeviceImportService extends BaseEntityImportService { private final DeviceService deviceService; - private final TbClusterService clusterService; @Override protected void setOwner(TenantId tenantId, Device device, NewIdProvider idProvider) { @@ -60,9 +56,9 @@ public class DeviceImportService extends BaseEntityImportService { + return super.getCallback(user, savedDevice, oldDevice).andThen(() -> { clusterService.onDeviceUpdated(savedDevice, oldDevice); - }; + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index f9d8d2ddd1..8db608fb42 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -19,20 +19,26 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; import org.thingsboard.server.utils.ThrowingRunnable; +import java.util.Objects; + @Service @TbCoreComponent @RequiredArgsConstructor public class DeviceProfileImportService extends BaseEntityImportService { private final DeviceProfileService deviceProfileService; + private final OtaPackageStateService otaPackageStateService; @Override protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, NewIdProvider idProvider) { @@ -50,9 +56,16 @@ public class DeviceProfileImportService extends BaseEntityImportService { - - }; + return super.getCallback(user, savedDeviceProfile, oldDeviceProfile).andThen(() -> { + clusterService.onDeviceProfileChange(savedDeviceProfile, null); + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + otaPackageStateService.update(savedDeviceProfile, + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index f6a96f24bf..ec8a0aa5a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -21,10 +21,13 @@ import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -83,9 +86,14 @@ public class RuleChainImportService extends BaseEntityImportService { - - }; + return super.getCallback(user, savedRuleChain, oldRuleChain).andThen(() -> { + if (savedRuleChain.getType() == RuleChainType.CORE) { + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } + }); } @Override diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java index 6965a79d19..858ccdf422 100644 --- a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java +++ b/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java @@ -21,4 +21,11 @@ public interface ThrowingRunnable { void run() throws ThingsboardException; + default ThrowingRunnable andThen(ThrowingRunnable after) { + return () -> { + this.run(); + after.run(); + }; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index f114af6625..9c3c8d3ff3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.JavaType; import lombok.SneakyThrows; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; @@ -54,6 +55,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +@Ignore @DaoSqlTest public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { From 1b48a98fa091b1e85c18aa53905d6f8e47af1f1e Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 8 Apr 2022 13:53:28 +0300 Subject: [PATCH 019/156] Save relations after all other entities are saved on import; refactoring --- .../EntitiesExportImportController.java | 15 +-- .../DefaultEntitiesExportImportService.java | 56 +++++++--- .../sync/EntitiesExportImportService.java | 8 +- .../sync/importing/EntityImportService.java | 2 + .../{ => data}/EntityImportResult.java | 25 +++-- .../{ => data}/EntityImportSettings.java | 2 +- .../importing/data/request/ImportRequest.java | 2 +- .../importing/impl/AssetImportService.java | 12 +-- .../impl/BaseEntityImportService.java | 102 +++++++++--------- .../importing/impl/CustomerImportService.java | 14 +-- .../impl/DashboardImportService.java | 12 +-- .../importing/impl/DeviceImportService.java | 8 +- .../impl/DeviceProfileImportService.java | 22 ++-- .../impl/RuleChainImportService.java | 18 ++-- ...EntitiesExportImportControllerSqlTest.java | 2 +- .../java/org/thingsboard/server/dao/Dao.java | 2 + .../server/dao/asset/BaseAssetService.java | 2 +- .../server/dao/sql/JpaAbstractDao.java | 8 ++ 18 files changed, 169 insertions(+), 143 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/importing/{ => data}/EntityImportResult.java (60%) rename application/src/main/java/org/thingsboard/server/service/sync/importing/{ => data}/EntityImportSettings.java (94%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 366eaec41a..e1f5dd4525 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -46,16 +46,13 @@ import org.thingsboard.server.service.sync.exporting.data.request.EntityQueryExp import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.UUID; -import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @@ -163,15 +160,7 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List>> importResults = exportImportService.importEntities(user, - importRequest.getExportDataList(), importRequest.getImportSettings()); - - for (EntityImportResult> entityImportResult : importResults) { - if (entityImportResult.getCallback() != null) { - entityImportResult.getCallback().run(); - } - } - return importResults; + return exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 9bca5f8bc9..5df3cdde0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -35,12 +36,13 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.utils.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; @@ -48,10 +50,13 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor +@Slf4j public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); @@ -77,7 +82,37 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class) @Override - public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { + public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + List>> importResults = new ArrayList<>(); + + for (EntityExportData> exportData : exportDataList) { + EntityImportResult> importResult = importEntity(user, exportData, importSettings); + importResults.add(importResult); + } + + for (ThrowingRunnable saveReferencesCallback : importResults.stream() + .map(EntityImportResult::getSaveReferencesCallback) + .filter(Objects::nonNull) + .collect(Collectors.toList())) { + saveReferencesCallback.run(); + } + + importResults.stream() + .map(EntityImportResult::getPushEventsCallback) + .filter(Objects::nonNull) + .forEach(pushEventsCallback -> { + try { + pushEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + + return importResults; + } + + private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); } @@ -85,22 +120,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityType entityType = exportData.getEntityType(); EntityImportService> importService = getImportService(entityType); - // TODO [viacheslav]: might throw DataIntegrityViolationException with cause of ConstraintViolationException: need to give normal error return importService.importEntity(user, exportData, importSettings); } - @Transactional(rollbackFor = Exception.class) - @Override - public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); - - List>> importResults = new ArrayList<>(); - for (EntityExportData> exportData : exportDataList) { - importResults.add(importEntity(user, exportData, importSettings)); - } - return importResults; - } - @Override public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index 3fa9b13406..b0690ea9ca 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -19,10 +19,10 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.List; @@ -30,8 +30,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException; - List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java index e3e40eea2a..6c82530273 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java @@ -21,6 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java index 90d47c4378..e51048c433 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java @@ -13,19 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Data; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.utils.ThrowingRunnable; -@Data public class EntityImportResult> { + @Getter @Setter private E savedEntity; + @Getter @Setter private E oldEntity; - @JsonIgnore - private transient ThrowingRunnable callback; // to call when entity is successfully saved and transaction is committed + + @Getter @JsonIgnore + private ThrowingRunnable saveReferencesCallback = () -> {}; + @Getter @JsonIgnore + private ThrowingRunnable pushEventsCallback = () -> {}; + + public void addSaveReferencesCallback(ThrowingRunnable callback) { + this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); + } + + public void addPushEventsCallback(ThrowingRunnable callback) { + this.pushEventsCallback = this.pushEventsCallback.andThen(callback); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index 9145762bef..1b5a93c0d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.importing.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java index cc2eeba618..6e857958d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java @@ -19,7 +19,7 @@ import lombok.Data; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 9a79c26515..0c8587270a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -26,7 +26,6 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.AssetExportData; -import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -47,12 +46,11 @@ public class AssetImportService extends BaseEntityImportService { - if (oldAsset != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) { + super.onEntitySaved(user, savedAsset, oldAsset); + if (oldAsset != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 9715fe008e..b7602a53bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -37,10 +37,9 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.Collections; import java.util.LinkedList; @@ -81,12 +80,13 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); - importResult.setCallback(callback); + + processAfterSaved(user, importResult, exportData, idProvider, importSettings); + return importResult; } @@ -94,60 +94,66 @@ public abstract class BaseEntityImportService newRelations = new LinkedList<>(); + protected void processAfterSaved(SecurityUser user, EntityImportResult importResult, D exportData, + NewIdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { + E savedEntity = importResult.getSavedEntity(); + E oldEntity = importResult.getOldEntity(); - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - newRelations.addAll(exportData.getInboundRelations().stream() - .peek(relation -> relation.setTo(savedEntity.getId())) - .collect(Collectors.toList())); + importResult.addSaveReferencesCallback(() -> { + List newRelations = new LinkedList<>(); - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); - relationService.deleteRelation(user.getTenantId(), existingRelation); + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + newRelations.addAll(exportData.getInboundRelations().stream() + .peek(relation -> relation.setTo(savedEntity.getId())) + .collect(Collectors.toList())); + + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { + for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } - } - if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { - newRelations.addAll(exportData.getOutboundRelations().stream() - .peek(relation -> relation.setFrom(savedEntity.getId())) - .collect(Collectors.toList())); - - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); - relationService.deleteRelation(user.getTenantId(), existingRelation); + if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { + newRelations.addAll(exportData.getOutboundRelations().stream() + .peek(relation -> relation.setFrom(savedEntity.getId())) + .collect(Collectors.toList())); + + if (importSettings.isRemoveExistingRelations() && oldEntity != null) { + for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { + exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); + relationService.deleteRelation(user.getTenantId(), existingRelation); + } } } - } - for (EntityRelation relation : newRelations) { - HasId otherEntity = null; - if (!relation.getTo().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); - relation.setTo(otherEntity.getId()); - } - if (!relation.getFrom().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); - relation.setFrom(otherEntity.getId()); - } - if (otherEntity != null) { - exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); - } + for (EntityRelation relation : newRelations) { + HasId otherEntity = null; + if (!relation.getTo().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); + relation.setTo(otherEntity.getId()); + } + if (!relation.getFrom().equals(savedEntity.getId())) { + otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); + relation.setFrom(otherEntity.getId()); + } + if (otherEntity != null) { + exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); + } - relationService.saveRelation(user.getTenantId(), relation); - } + relationService.saveRelation(user.getTenantId(), relation); + } + }); - return getCallback(user, savedEntity, oldEntity); + importResult.addPushEventsCallback(() -> { + onEntitySaved(user, savedEntity, oldEntity); + }); } - protected ThrowingRunnable getCallback(SecurityUser user, E savedEntity, E oldEntity) { - return () -> { - entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), - oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); - }; + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, + savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index a03ee60569..4d2cfdfffb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -20,15 +20,12 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -48,12 +45,11 @@ public class CustomerImportService extends BaseEntityImportService { - if (oldCustomer != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) { + super.onEntitySaved(user, savedCustomer, oldCustomer); + if (oldCustomer != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index e80e699c44..6463fe581c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -35,7 +35,6 @@ import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; -import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.HashSet; @@ -111,12 +110,11 @@ public class DashboardImportService extends BaseEntityImportService { - if (oldDashboard != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, Dashboard savedDashboard, Dashboard oldDashboard) { + super.onEntitySaved(user, savedDashboard, oldDashboard); + if (oldDashboard != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 7484fb98ba..850fea2baa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -25,7 +25,6 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.utils.ThrowingRunnable; @Service @TbCoreComponent @@ -55,10 +54,9 @@ public class DeviceImportService extends BaseEntityImportService { - clusterService.onDeviceUpdated(savedDevice, oldDevice); - }); + protected void onEntitySaved(SecurityUser user, Device savedDevice, Device oldDevice) { + super.onEntitySaved(user, savedDevice, oldDevice); + clusterService.onDeviceUpdated(savedDevice, oldDevice); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java index 8db608fb42..b68ae2a776 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; -import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Objects; @@ -55,17 +54,16 @@ public class DeviceProfileImportService extends BaseEntityImportService { - clusterService.onDeviceProfileChange(savedDeviceProfile, null); - clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), - oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), - oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); - otaPackageStateService.update(savedDeviceProfile, - oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), - oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); - }); + protected void onEntitySaved(SecurityUser user, DeviceProfile savedDeviceProfile, DeviceProfile oldDeviceProfile) { + super.onEntitySaved(user, savedDeviceProfile, oldDeviceProfile); + clusterService.onDeviceProfileChange(savedDeviceProfile, null); + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + otaPackageStateService.update(savedDeviceProfile, + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index ec8a0aa5a2..16f11171ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -32,7 +32,6 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.utils.ThrowingRunnable; import java.util.Collections; import java.util.Optional; @@ -85,15 +84,14 @@ public class RuleChainImportService extends BaseEntityImportService { - if (savedRuleChain.getType() == RuleChainType.CORE) { - clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), - oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); - } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { - entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); - } - }); + protected void onEntitySaved(SecurityUser user, RuleChain savedRuleChain, RuleChain oldRuleChain) { + super.onEntitySaved(user, savedRuleChain, oldRuleChain); + if (savedRuleChain.getType() == RuleChainType.CORE) { + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } } @Override diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 9c3c8d3ff3..603a43bf8c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -48,7 +48,7 @@ import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import java.util.List; diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 038be61d21..663c19348d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -37,6 +37,8 @@ public interface Dao { T save(TenantId tenantId, T t); + T saveAndFlush(TenantId tenantId, T t); + boolean removeById(TenantId tenantId, UUID id); void removeAllByIds(Collection ids); diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index bf8d75d27a..2c379ae91e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -117,7 +117,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ assetValidator.validate(asset, Asset::getTenantId); Asset savedAsset; try { - savedAsset = assetDao.save(asset.getTenantId(), asset); + savedAsset = assetDao.saveAndFlush(asset.getTenantId(), asset); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("asset_name_unq_key")) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 3c996ff6f5..ad6bb5ccec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -67,6 +67,14 @@ public abstract class JpaAbstractDao, D> return DaoUtil.getData(entity); } + @Override + @Transactional + public D saveAndFlush(TenantId tenantId, D domain) { + D d = save(tenantId, domain); + getRepository().flush(); + return d; + } + @Override public D findById(TenantId tenantId, UUID key) { log.debug("Get entity by key {}", key); From a35b16d7cb75d2fc73bc4696634fd1e931739587 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Fri, 8 Apr 2022 18:24:50 +0300 Subject: [PATCH 020/156] Get rid of boilerplate for export --- .../EntitiesExportImportController.java | 18 +++- .../DefaultEntitiesExportImportService.java | 40 +++++---- .../sync/exporting/EntityExportService.java | 3 - .../sync/exporting/data/AssetExportData.java | 32 ------- .../exporting/data/CustomerExportData.java | 32 ------- .../exporting/data/DashboardExportData.java | 32 ------- .../sync/exporting/data/DeviceExportData.java | 6 -- .../data/DeviceProfileExportData.java | 32 ------- .../sync/exporting/data/EntityExportData.java | 31 ++++--- .../exporting/data/RuleChainExportData.java | 6 -- .../data/request/EntityExportSettings.java | 6 ++ .../exporting/impl/AssetExportService.java | 39 -------- .../impl/BaseEntityExportService.java | 60 ++----------- .../exporting/impl/CustomerExportService.java | 39 -------- .../impl/DashboardExportService.java | 39 -------- .../impl/DefaultEntityExportService.java | 90 +++++++++++++++++++ .../exporting/impl/DeviceExportService.java | 6 +- .../impl/DeviceProfileExportService.java | 39 -------- .../impl/RuleChainExportService.java | 6 +- .../importing/data/EntityImportResult.java | 3 + .../importing/impl/AssetImportService.java | 9 +- .../impl/BaseEntityImportService.java | 3 +- .../importing/impl/CustomerImportService.java | 9 +- .../impl/DashboardImportService.java | 9 +- .../importing/impl/DeviceImportService.java | 3 +- .../impl/DeviceProfileImportService.java | 9 +- .../impl/RuleChainImportService.java | 3 +- ...aseEntitiesExportImportControllerTest.java | 62 ++++++++++++- ...EntitiesExportImportControllerSqlTest.java | 60 ++++--------- 29 files changed, 277 insertions(+), 449 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/AssetExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index e1f5dd4525..9f9e90b0ca 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -52,6 +53,7 @@ import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -61,6 +63,7 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @RequestMapping("/api/entities") @TbCoreComponent @RequiredArgsConstructor +@Slf4j public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; @@ -160,7 +163,20 @@ public class EntitiesExportImportController extends BaseController { public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - return exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + List>> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + + importResults.stream() + .map(EntityImportResult::getPushEventsCallback) + .filter(Objects::nonNull) + .forEach(pushEventsCallback -> { + try { + pushEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + + return importResults; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 5df3cdde0c..eefa396d06 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -39,6 +38,8 @@ import org.thingsboard.server.service.sync.exporting.EntityExportService; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; @@ -56,7 +57,6 @@ import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor -@Slf4j public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { private final Map> exportServices = new HashMap<>(); @@ -98,17 +98,6 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS saveReferencesCallback.run(); } - importResults.stream() - .map(EntityImportResult::getPushEventsCallback) - .filter(Objects::nonNull) - .forEach(pushEventsCallback -> { - try { - pushEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send event for entity", e); - } - }); - return importResults; } @@ -188,16 +177,31 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return daos.get(entityType); } + @Autowired - private void setServices(Collection> exportServices, - Collection> importServices, - Collection> daos) { - exportServices.forEach(entityExportService -> { - this.exportServices.put(entityExportService.getEntityType(), entityExportService); + private void setExportServices(DefaultEntityExportService defaultExportService, + Collection> exportServices) { + exportServices.stream() + .sorted(Comparator.comparing(exportService -> exportService.getSupportedEntityTypes().size(), Comparator.reverseOrder())) + .forEach(exportService -> { + exportService.getSupportedEntityTypes().forEach(entityType -> { + this.exportServices.put(entityType, exportService); + }); + }); + SUPPORTED_ENTITY_TYPES.forEach(entityType -> { + this.exportServices.putIfAbsent(entityType, defaultExportService); }); + } + + @Autowired + private void setImportServices(Collection> importServices) { importServices.forEach(entityImportService -> { this.importServices.put(entityImportService.getEntityType(), entityImportService); }); + } + + @Autowired + private void setDaos(Collection> daos) { daos.forEach(dao -> { if (dao.getEntityType() != null) { this.daos.put(dao.getEntityType(), dao); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java index c09cbcd36c..73f946e722 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.exporting; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -27,6 +26,4 @@ public interface EntityExportService { - - @Override - public EntityType getEntityType() { - return EntityType.ASSET; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java deleted file mode 100644 index 08888f4a35..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/CustomerExportData.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.data; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; - -@EqualsAndHashCode(callSuper = true) -@Data -public class CustomerExportData extends EntityExportData { - - @Override - public EntityType getEntityType() { - return EntityType.CUSTOMER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java deleted file mode 100644 index 6f101816c9..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DashboardExportData.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.data; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.EntityType; - -@EqualsAndHashCode(callSuper = true) -@Data -public class DashboardExportData extends EntityExportData { - - @Override - public EntityType getEntityType() { - return EntityType.DASHBOARD; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java index 977bd22f82..30d84c0a44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) @@ -27,9 +26,4 @@ public class DeviceExportData extends EntityExportData { private DeviceCredentials credentials; - @Override - public EntityType getEntityType() { - return EntityType.DEVICE; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java deleted file mode 100644 index 7b71749be4..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceProfileExportData.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.data; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.EntityType; - -@EqualsAndHashCode(callSuper = true) -@Data -public class DeviceProfileExportData extends EntityExportData { - - @Override - public EntityType getEntityType() { - return EntityType.DEVICE_PROFILE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index 51b5615c9e..bf44b0dc37 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -15,39 +15,48 @@ */ package org.thingsboard.server.service.sync.exporting.data; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.rule.RuleChain; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class, visible = true) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), - @Type(name = "DEVICE_PROFILE", value = DeviceProfileExportData.class), - @Type(name = "ASSET", value = AssetExportData.class), - @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), - @Type(name = "DASHBOARD", value = DashboardExportData.class), - @Type(name = "CUSTOMER", value = CustomerExportData.class) + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) }) @JsonInclude(JsonInclude.Include.NON_NULL) @Data -public abstract class EntityExportData> { +public class EntityExportData> { + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) + @JsonSubTypes({ + @Type(name = "DEVICE", value = Device.class), + @Type(name = "RULE_CHAIN", value = RuleChain.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), + @Type(name = "ASSET", value = Asset.class), + @Type(name = "DASHBOARD", value = Dashboard.class), + @Type(name = "CUSTOMER", value = Customer.class) + }) private E entity; + private EntityType entityType; + private List inboundRelations; private List outboundRelations; - @JsonIgnore - public abstract EntityType getEntityType(); - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java index 9f59aa4401..025459f27e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -27,9 +26,4 @@ public class RuleChainExportData extends EntityExportData { private RuleChainMetaData metaData; - @Override - public EntityType getEntityType() { - return EntityType.RULE_CHAIN; - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java index 01e4f8d068..ddf2e43bcf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java @@ -15,9 +15,15 @@ */ package org.thingsboard.server.service.sync.exporting.data.request; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@AllArgsConstructor +@NoArgsConstructor +@Builder public class EntityExportSettings { private boolean exportInboundRelations; private boolean exportOutboundRelations; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java deleted file mode 100644 index b4f506a1f5..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/AssetExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.AssetId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.AssetExportData; - -@Service -@TbCoreComponent -public class AssetExportService extends BaseEntityExportService { - - @Override - protected AssetExportData newExportData() { - return new AssetExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.ASSET; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java index 5d693e25b6..7b5f758f93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java @@ -15,71 +15,29 @@ */ package org.thingsboard.server.service.sync.exporting.impl; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; -import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import java.util.List; - -public abstract class BaseEntityExportService, D extends EntityExportData> implements EntityExportService { +import java.util.Set; - @Autowired @Lazy - private ExportableEntitiesService exportableEntitiesService; - @Autowired - private RelationService relationService; +public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @Override - public final D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - D exportData = newExportData(); - - E entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); - if (entity == null) { - throw new IllegalArgumentException("Entity not found"); - } - exportableEntitiesService.checkPermission(user, entity, getEntityType(), Operation.READ); - - exportData.setEntity(entity); - setRelatedEntities(user.getTenantId(), entity, exportData); - setAdditionalExportData(user, entity, exportData, exportSettings); - - return exportData; + protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + setRelatedEntities(user.getTenantId(), entity, (D) exportData); + super.setAdditionalExportData(user, entity, exportData, exportSettings); } protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData) {} - protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { - if (exportSettings.isExportInboundRelations()) { - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (inboundRelations != null) { - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } - } - exportData.setInboundRelations(inboundRelations); - } - if (exportSettings.isExportOutboundRelations()) { - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (outboundRelations != null) { - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } - } - exportData.setOutboundRelations(outboundRelations); - } - } - protected abstract D newExportData(); + public abstract Set getSupportedEntityTypes(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java deleted file mode 100644 index c90d21e573..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/CustomerExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.CustomerId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.CustomerExportData; - -@Service -@TbCoreComponent -public class CustomerExportService extends BaseEntityExportService { - - @Override - protected CustomerExportData newExportData() { - return new CustomerExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.CUSTOMER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java deleted file mode 100644 index 32584ca917..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DashboardExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; - -@Service -@TbCoreComponent -public class DashboardExportService extends BaseEntityExportService { - - @Override - protected DashboardExportData newExportData() { - return new DashboardExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.DASHBOARD; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java new file mode 100644 index 0000000000..4fe09e9322 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; + +import java.util.List; + +@Service +@TbCoreComponent +@Primary +public class DefaultEntityExportService, D extends EntityExportData> implements EntityExportService { + + @Autowired @Lazy + private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + + @Override + public final D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + D exportData = newExportData(); + + E entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (entity == null) { + throw new IllegalArgumentException(entityId.getEntityType() + " [" + entityId.getId() + "] not found"); + } + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); + + exportData.setEntity(entity); + exportData.setEntityType(entityId.getEntityType()); + setAdditionalExportData(user, entity, exportData, exportSettings); + + return exportData; + } + + protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + if (exportSettings.isExportInboundRelations()) { + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + if (inboundRelations != null) { + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + } + exportData.setInboundRelations(inboundRelations); + } + if (exportSettings.isExportOutboundRelations()) { + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + if (outboundRelations != null) { + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); + } + } + exportData.setOutboundRelations(outboundRelations); + } + } + + protected D newExportData() { + return (D) new EntityExportData(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java index d31a859013..da1e4f3a96 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java @@ -25,6 +25,8 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import java.util.Set; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -43,8 +45,8 @@ public class DeviceExportService extends BaseEntityExportService getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java deleted file mode 100644 index b3fd65363b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceProfileExportService.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.exporting.impl; - -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DeviceProfileExportData; - -@Service -@TbCoreComponent -public class DeviceProfileExportService extends BaseEntityExportService { - - @Override - protected DeviceProfileExportData newExportData() { - return new DeviceProfileExportData(); - } - - @Override - public EntityType getEntityType() { - return EntityType.DEVICE_PROFILE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java index 846d4006d0..e814cd6ce7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java @@ -25,6 +25,8 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import java.util.Set; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -43,8 +45,8 @@ public class RuleChainExportService extends BaseEntityExportService getSupportedEntityTypes() { + return Set.of(EntityType.RULE_CHAIN); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java index e51048c433..7160a483b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.utils.ThrowingRunnable; @@ -27,6 +28,8 @@ public class EntityImportResult> private E savedEntity; @Getter @Setter private E oldEntity; + @Getter @Setter + private EntityType entityType; @Getter @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 0c8587270a..02eaae27b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -20,17 +20,18 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.AssetExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; @Service @TbCoreComponent @RequiredArgsConstructor -public class AssetImportService extends BaseEntityImportService { +public class AssetImportService extends BaseEntityImportService> { private final AssetService assetService; @@ -41,12 +42,12 @@ public class AssetImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { return assetService.saveAsset(asset); } @Override - protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) { + protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) throws ThingsboardException { super.onEntitySaved(user, savedAsset, oldAsset); if (oldAsset != null) { entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index b7602a53bf..b576ae1e60 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -84,6 +84,7 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); importResult.setOldEntity(existingEntity); + importResult.setEntityType(getEntityType()); processAfterSaved(user, importResult, exportData, idProvider, importSettings); @@ -150,7 +151,7 @@ public abstract class BaseEntityImportService { +public class CustomerImportService extends BaseEntityImportService> { private final CustomerService customerService; @@ -40,12 +41,12 @@ public class CustomerImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { return customerService.saveCustomer(customer); } @Override - protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) { + protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException { super.onEntitySaved(user, savedCustomer, oldCustomer); if (oldCustomer != null) { entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 6463fe581c..3e06480a2f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,7 +35,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.DashboardExportData; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import java.util.Collections; import java.util.HashSet; @@ -46,7 +47,7 @@ import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor -public class DashboardImportService extends BaseEntityImportService { +public class DashboardImportService extends BaseEntityImportService> { private final DashboardService dashboardService; @@ -58,7 +59,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { Optional.ofNullable(dashboard.getConfiguration()) .flatMap(configuration -> Optional.ofNullable(configuration.get("entityAliases"))) .filter(JsonNode::isObject) @@ -110,7 +111,7 @@ public class DashboardImportService extends BaseEntityImportService { +public class DeviceProfileImportService extends BaseEntityImportService> { private final DeviceProfileService deviceProfileService; private final OtaPackageStateService otaPackageStateService; @@ -45,7 +46,7 @@ public class DeviceProfileImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { deviceProfile.setDefaultRuleChainId(idProvider.get(DeviceProfile::getDefaultRuleChainId)); deviceProfile.setDefaultDashboardId(idProvider.get(DeviceProfile::getDefaultDashboardId)); deviceProfile.setFirmwareId(idProvider.get(DeviceProfile::getFirmwareId)); @@ -54,7 +55,7 @@ public class DeviceProfileImportService extends BaseEntityImportService, I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { + SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + exportRequest.setEntityId(entityId); + exportRequest.setExportSettings(new EntityExportSettings()); + return (EntityExportData) exportEntities(exportRequest).get(0); + } + + protected List>> exportEntities(ExportRequest exportRequest) throws Exception { + return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>>() {}); + } + + protected List>> exportEntities(List exportRequests) throws Exception { + return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>>() {}); + } + + + protected , I extends EntityId> EntityImportResult importEntity(EntityExportData exportData) throws Exception { + return (EntityImportResult) importEntities(List.of((EntityExportData>)exportData)).get(0); + } + + protected List>> importEntities(List>> exportDataList) throws Exception { + ImportRequest importRequest = new ImportRequest(); + importRequest.setImportSettings(EntityImportSettings.builder() + .updateReferencesToOtherEntities(true) + .build()); + importRequest.setExportDataList(exportDataList); + return getResponse(doPost("/api/entities/import", importRequest), new TypeReference>>>() { + @Override + public Type getType() { + return mapper.getTypeFactory().constructCollectionType(List.class, + mapper.getTypeFactory().constructParametricType(EntityImportResult.class, + exportDataList.get(0).getEntity().getClass())); + } + }); + } + + protected T getResponse(ResultActions resultActions, TypeReference typeReference) throws Exception { + try { + return readResponse(resultActions.andExpect(status().isOk()), typeReference); + } catch (AssertionError e) { + throw new AssertionError(readResponse(resultActions, String.class), e); + } + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 603a43bf8c..ccbf311bef 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -15,15 +15,10 @@ */ package org.thingsboard.server.controller.sql; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JavaType; -import lombok.SneakyThrows; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -33,7 +28,7 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -50,12 +45,8 @@ import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import java.util.List; - import static org.assertj.core.api.Assertions.assertThat; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@Ignore @DaoSqlTest public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { @@ -110,7 +101,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(exportData.getEntity()).isEqualTo(asset); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, asset, tenantId2, importResult); checkImportedAssetData(asset, importResult.getSavedEntity()); @@ -122,7 +113,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); EntityExportData exportData = exportSingleEntity(asset.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, asset, tenantId1, importResult); checkImportedAssetData(asset, importResult.getSavedEntity()); @@ -138,8 +129,8 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData assetExportData = exportSingleEntity(asset.getId()); logInAsTenantAdmin2(); - Customer importedCustomer = importEntities(List.of(customerExportData)).get(0).getSavedEntity(); - Asset importedAsset = importEntities(List.of(assetExportData)).get(0).getSavedEntity(); + Customer importedCustomer = importEntity(customerExportData).getSavedEntity(); + Asset importedAsset = importEntity(assetExportData).getSavedEntity(); assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); } @@ -150,7 +141,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Customer customer = createCustomer(tenantId1, "My customer"); Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); - Asset importedAsset = this.importEntities(List.of(exportSingleEntity(asset.getId()))).get(0).getSavedEntity(); + Asset importedAsset = importEntity(this.exportSingleEntity(asset.getId())).getSavedEntity(); assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); } @@ -171,7 +162,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(exportData.getEntity()).isEqualTo(customer); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, customer, tenantId2, importResult); checkImportedCustomerData(customer, importResult.getSavedEntity()); @@ -183,7 +174,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); EntityExportData exportData = exportSingleEntity(customer.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, customer, tenantId1, importResult); checkImportedCustomerData(customer, importResult.getSavedEntity()); @@ -214,12 +205,12 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); logInAsTenantAdmin2(); - EntityImportResult profileImportResult = importEntities(List.of(profileExportData)).get(0); + EntityImportResult profileImportResult = importEntity(profileExportData); checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult); checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); - EntityImportResult deviceImportResult = importEntities(List.of(deviceExportData)).get(0); + EntityImportResult deviceImportResult = importEntity(deviceExportData); Device importedDevice = deviceImportResult.getSavedEntity(); checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult); checkImportedDeviceData(device, importedDevice); @@ -242,7 +233,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData deviceExportData = exportSingleEntity(device.getId()); - EntityImportResult importResult = importEntities(List.of(deviceExportData)).get(0); + EntityImportResult importResult = importEntity(deviceExportData); Device importedDevice = importResult.getSavedEntity(); checkImportedEntity(tenantId1, device, tenantId1, importResult); @@ -277,7 +268,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(((RuleChainExportData) exportData).getMetaData()).isEqualTo(metaData); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); @@ -293,7 +284,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData exportData = exportSingleEntity(ruleChain.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); @@ -330,7 +321,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(exportData.getEntity()).isEqualTo(dashboard); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, dashboard, tenantId2, importResult); checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } @@ -342,7 +333,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityExportData exportData = exportSingleEntity(dashboard.getId()); - EntityImportResult importResult = importEntities(List.of(exportData)).get(0); + EntityImportResult importResult = importEntity(exportData); checkImportedEntity(tenantId1, dashboard, tenantId1, importResult); checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } @@ -376,27 +367,6 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } - @SneakyThrows - private > List> importEntities(List> exportDataList) { - String requestJson = mapper.writerFor(new TypeReference>>() {}).writeValueAsString(exportDataList); - ResultActions resultActions = doPost("/api/entities/import", (Object) requestJson); - - try { - String responseJson = resultActions.andExpect(status().isOk()).andReturn().getResponse().getContentAsString(); - JavaType type = mapper.getTypeFactory().constructCollectionType(List.class, - mapper.getTypeFactory().constructParametricType(EntityImportResult.class, exportDataList.get(0).getEntity().getClass())); - return mapper.readValue(responseJson, type); - } catch (AssertionError e) { - throw new AssertionError(readResponse(resultActions, String.class), e); - } - } - - private , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { - return readResponse(doPost("/api/entities/export/" + entityId.getEntityType() + "/" + entityId.getId().toString()) - .andExpect(status().isOk()), new TypeReference>() {}); - } - - private void logInAsTenantAdmin1() throws Exception { login(tenantAdmin1.getEmail(), "12345678"); } From c04a0ff3a6fbef52c93a40bd56a9eb26fb3aea63 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 12 Apr 2022 13:04:25 +0300 Subject: [PATCH 021/156] More tests for export/import api; improvements --- .../EntitiesExportImportController.java | 45 +- .../DefaultEntitiesExportImportService.java | 10 +- .../sync/EntitiesExportImportService.java | 2 +- .../sync/exporting/data/EntityExportData.java | 19 +- ...a => CustomEntityFilterExportRequest.java} | 4 +- ...va => CustomEntityQueryExportRequest.java} | 4 +- .../exporting/data/request/ExportRequest.java | 6 +- .../data/request/ExportRequestType.java | 4 +- .../importing/data/EntityImportResult.java | 21 +- .../importing/data/EntityImportSettings.java | 2 + .../importing/data/request/ImportRequest.java | 2 +- .../impl/BaseEntityImportService.java | 33 +- .../impl/DashboardImportService.java | 4 +- .../impl/RuleChainImportService.java | 4 +- .../server/utils/JsonTbEntity.java | 47 ++ ...aseEntitiesExportImportControllerTest.java | 197 ++++- ...EntitiesExportImportControllerSqlTest.java | 731 ++++++++++++++---- .../server/common/data/ExportableEntity.java | 3 + 18 files changed, 875 insertions(+), 263 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/{EntityFilterExportRequest.java => CustomEntityFilterExportRequest.java} (89%) rename application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/{EntityQueryExportRequest.java => CustomEntityQueryExportRequest.java} (89%) create mode 100644 application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 9f9e90b0ca..4a1badd38e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,13 +17,13 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DataAccessException; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -41,9 +41,9 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityQueryExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; @@ -71,7 +71,7 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/export") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { + public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { return exportEntitiesByRequest(user, exportRequest); @@ -82,10 +82,10 @@ public class EntitiesExportImportController extends BaseController { @PostMapping(value = "/export", params = {"multiple"}) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { + public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List>> exportDataList = new ArrayList<>(); + List> exportDataList = new ArrayList<>(); for (ExportRequest exportRequest : exportRequests) { exportDataList.addAll(exportEntitiesByRequest(user, exportRequest)); } @@ -96,10 +96,10 @@ public class EntitiesExportImportController extends BaseController { } - private List>> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { + private List> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { List entitiesIds = findEntitiesForRequest(user, request); - List>> exportDataList = new ArrayList<>(); + List> exportDataList = new ArrayList<>(); for (EntityId entityId : entitiesIds) { exportDataList.add(exportImportService.exportEntity(user, entityId, request.getExportSettings())); } @@ -122,15 +122,15 @@ public class EntitiesExportImportController extends BaseController { CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); return findEntitiesByFilter(user.getTenantId(), customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); } - case ENTITY_FILTER: { - EntityFilterExportRequest exportRequest = (EntityFilterExportRequest) request; + case CUSTOM_ENTITY_FILTER: { + CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; EntityFilter filter = exportRequest.getFilter(); CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); return findEntitiesByFilter(user.getTenantId(), customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); } - case ENTITY_QUERY:{ - EntityQueryExportRequest exportRequest = (EntityQueryExportRequest) request; + case CUSTOM_ENTITY_QUERY: { + CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; EntityDataQuery query = exportRequest.getQuery(); CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); @@ -153,24 +153,29 @@ public class EntitiesExportImportController extends BaseController { } private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { - return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() - .map(EntityData::getEntityId) - .collect(Collectors.toList()); + try { + return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + .map(EntityData::getEntityId) + .collect(Collectors.toList()); + } catch (DataAccessException e) { + log.error("Failed to find entity data by query: {}", e.getMessage()); + throw new IllegalArgumentException("Entity filter cannot be processed"); + } } @PostMapping("/import") - public List>> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { + public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List>> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); importResults.stream() - .map(EntityImportResult::getPushEventsCallback) + .map(EntityImportResult::getSendEventsCallback) .filter(Objects::nonNull) - .forEach(pushEventsCallback -> { + .forEach(sendEventsCallback -> { try { - pushEventsCallback.run(); + sendEventsCallback.run(); } catch (Exception e) { log.error("Failed to send event for entity", e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index eefa396d06..cfb895c36a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -67,7 +67,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, - EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD + EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE ); @@ -82,12 +82,12 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class) @Override - public List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { + public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); - List>> importResults = new ArrayList<>(); + List> importResults = new ArrayList<>(); - for (EntityExportData> exportData : exportDataList) { - EntityImportResult> importResult = importEntity(user, exportData, importSettings); + for (EntityExportData exportData : exportDataList) { + EntityImportResult importResult = importEntity(user, exportData, importSettings); importResults.add(importResult); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java index b0690ea9ca..eaaf01ba2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java @@ -30,6 +30,6 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; - List>> importEntities(SecurityUser user, List>> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; + List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index bf44b0dc37..4ab6dd57b1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -21,21 +21,16 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Dashboard; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.utils.JsonTbEntity; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class, visible = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) @@ -44,15 +39,7 @@ import java.util.List; @Data public class EntityExportData> { - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) - @JsonSubTypes({ - @Type(name = "DEVICE", value = Device.class), - @Type(name = "RULE_CHAIN", value = RuleChain.class), - @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), - @Type(name = "ASSET", value = Asset.class), - @Type(name = "DASHBOARD", value = Dashboard.class), - @Type(name = "CUSTOMER", value = Customer.class) - }) + @JsonTbEntity private E entity; private EntityType entityType; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java index 00ef0604e5..02cbca98d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.query.EntityFilter; @EqualsAndHashCode(callSuper = true) @Data -public class EntityFilterExportRequest extends ExportRequest { +public class CustomEntityFilterExportRequest extends ExportRequest { private EntityFilter filter; private int page; @@ -31,7 +31,7 @@ public class EntityFilterExportRequest extends ExportRequest { @Override public ExportRequestType getType() { - return ExportRequestType.ENTITY_FILTER; + return ExportRequestType.CUSTOM_ENTITY_FILTER; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java index 85232821d7..76d258e891 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java @@ -22,14 +22,14 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; @EqualsAndHashCode(callSuper = true) @Data -public class EntityQueryExportRequest extends ExportRequest { +public class CustomEntityQueryExportRequest extends ExportRequest { private EntityDataQuery query; private CustomerId customerId; @Override public ExportRequestType getType() { - return ExportRequestType.ENTITY_QUERY; + return ExportRequestType.CUSTOM_ENTITY_QUERY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java index c140786d06..af0f161227 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.exporting.data.request; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -27,15 +26,14 @@ import lombok.Data; @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), - @Type(name = "ENTITY_FILTER", value = EntityFilterExportRequest.class), - @Type(name = "ENTITY_QUERY", value = EntityQueryExportRequest.class) + @Type(name = "CUSTOM_ENTITY_FILTER", value = CustomEntityFilterExportRequest.class), + @Type(name = "CUSTOM_ENTITY_QUERY", value = CustomEntityQueryExportRequest.class) }) @Data public abstract class ExportRequest { private EntityExportSettings exportSettings; - @JsonIgnore public abstract ExportRequestType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java index 07e5c946ba..c36ac567e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java @@ -20,6 +20,6 @@ public enum ExportRequestType { ENTITY_LIST, ENTITY_TYPE, - ENTITY_FILTER, - ENTITY_QUERY + CUSTOM_ENTITY_FILTER, + CUSTOM_ENTITY_QUERY } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java index 7160a483b3..62c574d5f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java @@ -16,32 +16,33 @@ package org.thingsboard.server.service.sync.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; -import lombok.Setter; +import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.utils.JsonTbEntity; import org.thingsboard.server.utils.ThrowingRunnable; +@Data public class EntityImportResult> { - @Getter @Setter + + @JsonTbEntity private E savedEntity; - @Getter @Setter + @JsonTbEntity private E oldEntity; - @Getter @Setter private EntityType entityType; - @Getter @JsonIgnore + @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; - @Getter @JsonIgnore - private ThrowingRunnable pushEventsCallback = () -> {}; + @JsonIgnore + private ThrowingRunnable sendEventsCallback = () -> {}; public void addSaveReferencesCallback(ThrowingRunnable callback) { this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); } - public void addPushEventsCallback(ThrowingRunnable callback) { - this.pushEventsCallback = this.pushEventsCallback.andThen(callback); + public void addSendEventsCallback(ThrowingRunnable callback) { + this.sendEventsCallback = this.sendEventsCallback.andThen(callback); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index 1b5a93c0d8..3b457e1f5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -26,8 +26,10 @@ import lombok.NoArgsConstructor; @Builder public class EntityImportSettings { private boolean findExistingByName; + private boolean importInboundRelations; private boolean importOutboundRelations; private boolean removeExistingRelations; + private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java index 6e857958d6..7943e1a404 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java @@ -26,7 +26,7 @@ import java.util.List; @Data public class ImportRequest { - private List>> exportDataList; + private List> exportDataList; private EntityImportSettings importSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index b576ae1e60..97547868d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -21,7 +21,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.audit.ActionType; @@ -76,6 +75,7 @@ public abstract class BaseEntityImportService { + importResult.addSendEventsCallback(() -> { onEntitySaved(user, savedEntity, oldEntity); }); } @@ -172,8 +172,6 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { - if (externalId == null || externalId.isNullUid()) return null; - return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); @@ -187,19 +185,18 @@ public abstract class BaseEntityImportService ALWAYS_UPDATE_REFERENCED_IDS = Set.of( - EntityType.RULE_CHAIN - ); - public ID get(Function idExtractor) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities() - || ALWAYS_UPDATE_REFERENCED_IDS.contains(getEntityType())) { + if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { return getInternalId(idExtractor.apply(this.entity)); } else { return idExtractor.apply(existingEntity); } } + public ID getInternal(ID externalId) { + return getInternalId(externalId); + } + public Set get(Function> listExtractor, Function idGetter, BiConsumer idSetter) { if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() @@ -213,17 +210,15 @@ public abstract class BaseEntityImportService ID getInternalId(ID externalId) { + if (externalId == null || externalId.isNullUid()) return null; + HasId entity = findInternalEntity(user.getTenantId(), externalId); - if (entity != null) { - try { - exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); - } catch (ThingsboardException e) { - throw new IllegalArgumentException(e.getMessage(), e); - } - return entity.getId(); - } else { - return null; + try { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ); + } catch (ThingsboardException e) { + throw new IllegalArgumentException(e.getMessage(), e); } + return entity.getId(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index 3e06480a2f..fda6904421 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -74,7 +74,7 @@ public class DashboardImportService extends BaseEntityImportService { String uuid = matchResult.group(); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); - return idProvider.get(d -> entityId).toString(); + return idProvider.getInternal(entityId).toString(); }); ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); }); @@ -86,7 +86,7 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index f901fcfd17..5fef00cbed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -63,14 +63,14 @@ public class RuleChainImportService extends BaseEntityImportService { ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.get(rc -> new RuleChainId(otherRuleChainUuid)).toString() + idProvider.getInternal(new RuleChainId(otherRuleChainUuid)).toString() )); ruleNode.setConfiguration(ruleNodeConfig); }); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { - ruleChainConnectionInfo.setTargetRuleChainId(idProvider.get(rc -> ruleChainConnectionInfo.getTargetRuleChainId())); + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternal(ruleChainConnectionInfo.getTargetRuleChainId())); }); ruleChain.setFirstRuleNodeId(null); diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java b/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java new file mode 100644 index 0000000000..cfc38ff3c3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils; + +import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.rule.RuleChain; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +@JacksonAnnotationsInside +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) +@JsonSubTypes({ + @Type(name = "DEVICE", value = Device.class), + @Type(name = "RULE_CHAIN", value = RuleChain.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), + @Type(name = "ASSET", value = Asset.class), + @Type(name = "DASHBOARD", value = Dashboard.class), + @Type(name = "CUSTOMER", value = Customer.class) +}) +public @interface JsonTbEntity { +} diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index 0d73ed9f09..a8a5bca534 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -17,6 +17,8 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.JacksonUtil; @@ -28,6 +30,10 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; @@ -35,19 +41,29 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileCon import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; @@ -56,11 +72,12 @@ import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; -import java.lang.reflect.Type; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public abstract class BaseEntitiesExportImportControllerTest extends AbstractControllerTest { @@ -68,6 +85,8 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon @Autowired protected DeviceService deviceService; @Autowired + protected OtaPackageService otaPackageService; + @Autowired protected DeviceProfileService deviceProfileService; @Autowired protected AssetService assetService; @@ -77,6 +96,45 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected RuleChainService ruleChainService; @Autowired protected DashboardService dashboardService; + @Autowired + protected RelationService relationService; + @Autowired + protected TenantService tenantService; + + protected TenantId tenantId1; + protected User tenantAdmin1; + + protected TenantId tenantId2; + protected User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { Device device = new Device(); @@ -91,13 +149,38 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return deviceService.saveDevice(device); } - protected DeviceProfile createDeviceProfile(TenantId tenantId, String name) { + protected OtaPackage createOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type) { + OtaPackage otaPackage = new OtaPackage(); + otaPackage.setTenantId(tenantId); + otaPackage.setDeviceProfileId(deviceProfileId); + otaPackage.setType(type); + otaPackage.setTitle("My " + type); + otaPackage.setVersion("v1.0"); + otaPackage.setFileName("filename.txt"); + otaPackage.setContentType("text/plain"); + otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); + otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); + otaPackage.setDataSize(1L); + otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1})); + return otaPackageService.saveOtaPackage(otaPackage); + } + + protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, RuleChainId defaultRuleChainId, DashboardId defaultDashboardId, String name) { DeviceProfile deviceProfile = new DeviceProfile(); deviceProfile.setTenantId(tenantId); deviceProfile.setName(name); deviceProfile.setDescription("dscrptn"); deviceProfile.setType(DeviceProfileType.DEFAULT); deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + deviceProfile.setDefaultRuleChainId(defaultRuleChainId); + deviceProfile.setDefaultDashboardId(defaultDashboardId); DeviceProfileData profileData = new DeviceProfileData(); profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); @@ -105,6 +188,14 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return deviceProfileService.saveDeviceProfile(deviceProfile); } + protected void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { Asset asset = new Asset(); asset.setTenantId(tenantId); @@ -116,6 +207,13 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return assetService.saveAsset(asset); } + protected void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + protected Customer createCustomer(TenantId tenantId, String name) { Customer customer = new Customer(); customer.setTenantId(tenantId); @@ -127,6 +225,13 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return customerService.saveCustomer(customer); } + protected void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { Dashboard dashboard = new Dashboard(); dashboard.setTenantId(tenantId); @@ -142,6 +247,16 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return dashboard; } + protected void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + protected RuleChain createRuleChain(TenantId tenantId, String name) { RuleChain ruleChain = new RuleChain(); ruleChain.setTenantId(tenantId); @@ -178,6 +293,50 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); } + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + protected EntityRelation createRelation(EntityId from, EntityId to) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(from); + relation.setTo(to); + relation.setType(EntityRelation.MANAGES_TYPE); + relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(TenantId.SYS_TENANT_ID, relation); + return relation; + } + + protected & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + } + } + protected , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); @@ -186,33 +345,30 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon return (EntityExportData) exportEntities(exportRequest).get(0); } - protected List>> exportEntities(ExportRequest exportRequest) throws Exception { - return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>>() {}); + protected List> exportEntities(ExportRequest exportRequest) throws Exception { + return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>() {}); } - protected List>> exportEntities(List exportRequests) throws Exception { - return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>>() {}); + protected List> exportEntities(List exportRequests) throws Exception { + return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>() {}); } protected , I extends EntityId> EntityImportResult importEntity(EntityExportData exportData) throws Exception { - return (EntityImportResult) importEntities(List.of((EntityExportData>)exportData)).get(0); + return (EntityImportResult) importEntities(List.of((EntityExportData>) exportData)).get(0); } - protected List>> importEntities(List>> exportDataList) throws Exception { + protected List> importEntities(List> exportDataList) throws Exception { ImportRequest importRequest = new ImportRequest(); importRequest.setImportSettings(EntityImportSettings.builder() .updateReferencesToOtherEntities(true) .build()); importRequest.setExportDataList(exportDataList); - return getResponse(doPost("/api/entities/import", importRequest), new TypeReference>>>() { - @Override - public Type getType() { - return mapper.getTypeFactory().constructCollectionType(List.class, - mapper.getTypeFactory().constructParametricType(EntityImportResult.class, - exportDataList.get(0).getEntity().getClass())); - } - }); + return importEntities(importRequest); + } + + protected List> importEntities(ImportRequest importRequest) throws Exception { + return getResponse(doPost("/api/entities/import", importRequest), new TypeReference>>() {}); } protected T getResponse(ResultActions resultActions, TypeReference typeReference) throws Exception { @@ -223,4 +379,13 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon } } + + protected void logInAsTenantAdmin1() throws Exception { + login(tenantAdmin1.getEmail(), "12345678"); + } + + protected void logInAsTenantAdmin2() throws Exception { + login(tenantAdmin2.getEmail(), "12345678"); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index ccbf311bef..920d91e93d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -15,86 +15,81 @@ */ package org.thingsboard.server.controller.sql; -import org.junit.After; -import org.junit.Before; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.Streams; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.testcontainers.shaded.org.apache.commons.lang.RandomStringUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleNode; -import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.controller.BaseEntitiesExportImportControllerTest; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.verify; @DaoSqlTest public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImportControllerTest { - @Autowired - private TenantService tenantService; @Autowired private DeviceCredentialsService deviceCredentialsService; - @Autowired - private ExportableEntitiesService exportableEntitiesService; - - private TenantId tenantId1; - private User tenantAdmin1; - - private TenantId tenantId2; - private User tenantAdmin2; - - @Before - public void beforeEach() throws Exception { - loginSysAdmin(); - Tenant tenant1 = new Tenant(); - tenant1.setTitle("Tenant 1"); - tenant1.setEmail("tenant1@thingsboard.org"); - this.tenantId1 = tenantService.saveTenant(tenant1).getId(); - User tenantAdmin1 = new User(); - tenantAdmin1.setTenantId(tenantId1); - tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); - this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); - Tenant tenant2 = new Tenant(); - tenant2.setTitle("Tenant 2"); - tenant2.setEmail("tenant2@thingsboard.org"); - this.tenantId2 = tenantService.saveTenant(tenant2).getId(); - User tenantAdmin2 = new User(); - tenantAdmin2.setTenantId(tenantId2); - tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); - tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); - this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); - } - - @After - public void afterEach() { - tenantService.deleteTenant(tenantId1); - tenantService.deleteTenant(tenantId2); - } - + @SpyBean + private EntityActionService entityActionService; + @SpyBean + private TbClusterService clusterService; + @SpyBean + private OtaPackageStateService otaPackageStateService; @Test - public void testExportImportSingleAsset_betweenTenants() throws Exception { + public void testExportImportAsset_betweenTenants() throws Exception { logInAsTenantAdmin1(); Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); EntityExportData exportData = exportSingleEntity(asset.getId()); @@ -103,40 +98,24 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, asset, tenantId2, importResult); + checkImportedEntity(tenantId1, asset, tenantId2, importResult.getSavedEntity()); checkImportedAssetData(asset, importResult.getSavedEntity()); } @Test - public void testExportImportSingleAsset_sameTenant() throws Exception { + public void testExportImportAsset_sameTenant() throws Exception { logInAsTenantAdmin1(); Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); EntityExportData exportData = exportSingleEntity(asset.getId()); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, asset, tenantId1, importResult); + checkImportedEntity(tenantId1, asset, tenantId1, importResult.getSavedEntity()); checkImportedAssetData(asset, importResult.getSavedEntity()); } @Test - public void testExportImportAsset_withCustomer_betweenTenants() throws Exception { - logInAsTenantAdmin1(); - Customer customer = createCustomer(tenantId1, "My customer"); - Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); - - EntityExportData customerExportData = exportSingleEntity(customer.getId()); - EntityExportData assetExportData = exportSingleEntity(asset.getId()); - - logInAsTenantAdmin2(); - Customer importedCustomer = importEntity(customerExportData).getSavedEntity(); - Asset importedAsset = importEntity(assetExportData).getSavedEntity(); - - assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); - } - - @Test - public void testExportImportAsset_withCustomer_sameTenant() throws Exception { + public void testExportImportAsset_sameTenant_withCustomer() throws Exception { logInAsTenantAdmin1(); Customer customer = createCustomer(tenantId1, "My customer"); Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); @@ -146,16 +125,9 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); } - private void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { - assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); - assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); - assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); - assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); - } - @Test - public void testExportImportSingleCustomer_betweenTenants() throws Exception { + public void testExportImportCustomer_betweenTenants() throws Exception { logInAsTenantAdmin1(); Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); EntityExportData exportData = exportSingleEntity(customer.getId()); @@ -164,34 +136,27 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, customer, tenantId2, importResult); + checkImportedEntity(tenantId1, customer, tenantId2, importResult.getSavedEntity()); checkImportedCustomerData(customer, importResult.getSavedEntity()); } @Test - public void testExportImportSingleCustomer_sameTenant() throws Exception { + public void testExportImportCustomer_sameTenant() throws Exception { logInAsTenantAdmin1(); Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); EntityExportData exportData = exportSingleEntity(customer.getId()); EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, customer, tenantId1, importResult); + checkImportedEntity(tenantId1, customer, tenantId1, importResult.getSavedEntity()); checkImportedCustomerData(customer, importResult.getSavedEntity()); } - private void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { - assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); - assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); - assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); - assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); - } - @Test public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { logInAsTenantAdmin1(); - DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile of tenant 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile of tenant 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); @@ -206,13 +171,13 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); EntityImportResult profileImportResult = importEntity(profileExportData); - checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult.getSavedEntity()); checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); EntityImportResult deviceImportResult = importEntity(deviceExportData); Device importedDevice = deviceImportResult.getSavedEntity(); - checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult.getSavedEntity()); checkImportedDeviceData(device, importedDevice); assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); @@ -227,8 +192,14 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp @Test public void testExportImportDevice_sameTenant() throws Exception { logInAsTenantAdmin1(); - DeviceProfile deviceProfile = createDeviceProfile(tenantId1, "Device profile v1.0"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile v1.0"); + OtaPackage firmware = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.FIRMWARE); + OtaPackage software = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.SOFTWARE); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + device.setFirmwareId(firmware.getId()); + device.setSoftwareId(software.getId()); + device = deviceService.saveDevice(device); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); EntityExportData deviceExportData = exportSingleEntity(device.getId()); @@ -236,29 +207,119 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityImportResult importResult = importEntity(deviceExportData); Device importedDevice = importResult.getSavedEntity(); - checkImportedEntity(tenantId1, device, tenantId1, importResult); + checkImportedEntity(tenantId1, device, tenantId1, importResult.getSavedEntity()); assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + assertThat(importedDevice.getFirmwareId()).isEqualTo(firmware.getId()); + assertThat(importedDevice.getSoftwareId()).isEqualTo(software.getId()); + } + + + @Test + public void testExportImportDashboard_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + assertThat(exportData.getEntity()).isEqualTo(dashboard); + + logInAsTenantAdmin2(); + EntityImportResult importResult = importEntity(exportData); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } - private void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { - assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); - assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); - assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); - assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); - assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + @Test + public void testExportImportDashboard_sameTenant() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + + EntityImportResult importResult = importEntity(exportData); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); } - private void checkImportedDeviceData(Device initialDevice, Device importedDevice) { - assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); - assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); - assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); - assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + @Test + public void testExportImportDashboard_betweenTenants_withCustomer_updated() throws Exception { + logInAsTenantAdmin1(); + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + + EntityExportData exportData = exportSingleEntity(dashboard.getId()); + + logInAsTenantAdmin2(); + Dashboard importedDashboard = (Dashboard) importEntities(List.of(exportData)).get(0).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "Customer 1"); + EntityExportData customerExportData = exportSingleEntity(customer.getId()); + dashboardService.assignDashboardToCustomer(tenantId1, dashboard.getId(), customer.getId()); + exportData = exportSingleEntity(dashboard.getId()); + + logInAsTenantAdmin2(); + Customer importedCustomer = (Customer) importEntities(List.of(customerExportData)).get(0).getSavedEntity(); + importedDashboard = (Dashboard) importEntities(List.of(exportData)).get(0).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } + + @Test + public void testExportImportDashboard_betweenTenants_withEntityAliases() throws Exception { + logInAsTenantAdmin1(); + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + asset1.getId().toString() + "\",\n" + + "\t\t\t\t\"" + asset2.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + dashboard = dashboardService.saveDashboard(dashboard); + + EntityTypeExportRequest assetsExportRequest = new EntityTypeExportRequest(); + assetsExportRequest.setEntityType(EntityType.ASSET); + assetsExportRequest.setPageSize(10); + assetsExportRequest.setExportSettings(new EntityExportSettings()); + EntityTypeExportRequest dashboardsExportRequest = new EntityTypeExportRequest(); + dashboardsExportRequest.setEntityType(EntityType.DASHBOARD); + dashboardsExportRequest.setPageSize(10); + dashboardsExportRequest.setExportSettings(new EntityExportSettings()); + List> exportDataList = exportEntities(List.of(assetsExportRequest, dashboardsExportRequest)); + + logInAsTenantAdmin2(); + Map>> importResults = importEntities(exportDataList).stream().collect(Collectors.groupingBy(EntityImportResult::getEntityType)); + Asset importedAsset1 = (Asset) importResults.get(EntityType.ASSET).get(0).getSavedEntity(); + Asset importedAsset2 = (Asset) importResults.get(EntityType.ASSET).get(1).getSavedEntity(); + Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).get(0).getSavedEntity(); + + Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .map(JsonNode::asText).collect(Collectors.toSet()); + assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); + assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); } @Test - public void testExportImportSingleRuleChain_betweenTenants() throws Exception { + public void testExportImportRuleChain_betweenTenants() throws Exception { logInAsTenantAdmin1(); RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); @@ -272,12 +333,12 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); - checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult); + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult.getSavedEntity()); checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); } @Test - public void testExportImportSingleRuleChain_sameTenant() throws Exception { + public void testExportImportRuleChain_sameTenant() throws Exception { logInAsTenantAdmin1(); RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); @@ -288,91 +349,439 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp RuleChain importedRuleChain = importResult.getSavedEntity(); RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); - checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult); + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult.getSavedEntity()); checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); } - private void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { - assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); - assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); - assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); - assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); - - assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); - assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); - for (int i = 0; i < initialMetaData.getNodes().size(); i++) { - RuleNode initialNode = initialMetaData.getNodes().get(i); - RuleNode importedNode = importedMetaData.getNodes().get(i); - assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); - assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); - assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); - assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); - assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); - } + + @Test + public void testExportImportBatch_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, customer.getId(), "A", "Customer 1 - Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Customer 1 - Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Customer 1 - Device 1"); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), ruleChain.getId(), deviceProfile.getId(), dashboard.getId())); + List> exportDataList = exportEntities(exportRequest); + exportRequest.setEntitiesIds(List.of(device.getId())); + DeviceExportData deviceExportData = (DeviceExportData) exportEntities(exportRequest).get(0); + deviceExportData.getCredentials().setCredentialsId(RandomStringUtils.randomAlphanumeric(10)); + exportDataList.add(deviceExportData); + + logInAsTenantAdmin2(); + ImportRequest importRequest = new ImportRequest(); + importRequest.setImportSettings(EntityImportSettings.builder() + .updateReferencesToOtherEntities(true) + .build()); + importRequest.setExportDataList(exportDataList); + Map> importResults = importEntities(importRequest).stream() + .collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Customer importedCustomer = (Customer) importResults.get(EntityType.CUSTOMER).getSavedEntity(); + checkImportedEntity(tenantId1, customer, tenantId2, importedCustomer); + + Asset importedAsset = (Asset) importResults.get(EntityType.ASSET).getSavedEntity(); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + assertThat(importedAsset.getCustomerId()).isEqualTo(importedCustomer.getId()); + + RuleChain importedRuleChain = (RuleChain) importResults.get(EntityType.RULE_CHAIN).getSavedEntity(); + checkImportedEntity(tenantId1, ruleChain, tenantId2, importedRuleChain); + + Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + assertThat(importedDashboard.getAssignedCustomers()).size().isOne(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importResults.get(EntityType.DEVICE_PROFILE).getSavedEntity(); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, importedDeviceProfile); + assertThat(importedDeviceProfile.getDefaultRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedDeviceProfile.getDefaultDashboardId()).isEqualTo(importedDashboard.getId()); + + Device importedDevice = (Device) importResults.get(EntityType.DEVICE).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + assertThat(importedDevice.getCustomerId()).isEqualTo(importedCustomer.getId()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(importedDeviceProfile.getId()); } @Test - public void testExportImportSingleDashboard_betweenTenants() throws Exception { + public void testExportImportWithInboundRelations_betweenTenants() throws Exception { logInAsTenantAdmin1(); - Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); - EntityExportData exportData = exportSingleEntity(dashboard.getId()); - assertThat(exportData.getEntity()).isEqualTo(dashboard); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportInboundRelations(true) + .exportOutboundRelations(false) + .build()); + List> exportDataList = exportEntities(exportRequest); + + EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); + assertThat(deviceExportData.getInboundRelations()).size().isOne(); + assertThat(deviceExportData.getInboundRelations().get(0)).matches(entityRelation -> { + return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); + }); + ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); logInAsTenantAdmin2(); - EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, dashboard, tenantId2, importResult); - checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(exportDataList); + importRequest.setImportSettings(EntityImportSettings.builder() + .importInboundRelations(true) + .build()); + Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Device importedDevice = (Device) importResults.get(EntityType.DEVICE).getSavedEntity(); + Asset importedAsset = (Asset) importResults.get(EntityType.ASSET).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); } @Test - public void testExportImportSingleDashboard_sameTenant() throws Exception { + public void testExportImportWithRelations_betweenTenants() throws Exception { logInAsTenantAdmin1(); - Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); - EntityExportData exportData = exportSingleEntity(dashboard.getId()); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityImportResult importResult = importEntity(exportData); - checkImportedEntity(tenantId1, dashboard, tenantId1, importResult); - checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportInboundRelations(true) + .exportOutboundRelations(true) + .build()); + List> exportDataList = exportEntities(exportRequest); + + assertThat(exportDataList).allMatch(exportData -> exportData.getInboundRelations().size() + exportData.getOutboundRelations().size() == 1); + + EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); + ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); + + logInAsTenantAdmin2(); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(exportDataList); + importRequest.setImportSettings(EntityImportSettings.builder() + .importInboundRelations(true) + .importOutboundRelations(true) + .build()); + Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Device importedDevice = (Device) importResults.get(EntityType.DEVICE).getSavedEntity(); + Asset importedAsset = (Asset) importResults.get(EntityType.ASSET).getSavedEntity(); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); } - private void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { - assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); - assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); - assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); - assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); - if (initialDashboard.getAssignedCustomers() != null) { - assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); - } + @Test + public void testExportImportWithRelations_sameTenant() throws Exception { + logInAsTenantAdmin1(); + + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device1 = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); + + SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + exportRequest.setEntityId(asset.getId()); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportOutboundRelations(true) + .build()); + EntityExportData assetExportData = (EntityExportData) exportEntities(exportRequest).get(0); + assertThat(assetExportData.getOutboundRelations()).size().isOne(); + + Device device2 = createDevice(tenantId1, null, null, "Device 2"); + EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(List.of(assetExportData)); + importRequest.setImportSettings(EntityImportSettings.builder() + .importOutboundRelations(true) + .build()); + + importEntities(importRequest); + + List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1, relation2); + } + + @Test + public void textExportImportWithRelations_sameTenant_removeExisting() throws Exception { + logInAsTenantAdmin1(); + + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); + + SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + exportRequest.setEntityId(device.getId()); + exportRequest.setExportSettings(EntityExportSettings.builder() + .exportInboundRelations(true) + .build()); + EntityExportData deviceExportData = exportEntities(exportRequest).get(0); + assertThat(deviceExportData.getInboundRelations()).size().isOne(); + + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); + + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(List.of(deviceExportData)); + importRequest.setImportSettings(EntityImportSettings.builder() + .importInboundRelations(true) + .removeExistingRelations(true) + .build()); + + importEntities(importRequest); + + List relations = relationService.findByTo(TenantId.SYS_TENANT_ID, device.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + + @Test + public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception { + logInAsTenantAdmin1(); + DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(defaultDeviceProfile.getId())); + exportRequest.setExportSettings(new EntityExportSettings()); + List> exportDataList = exportEntities(exportRequest); + + logInAsTenantAdmin2(); + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(exportDataList); + importRequest.setImportSettings(EntityImportSettings.builder() + .findExistingByName(false) + .updateReferencesToOtherEntities(true) + .build()); + assertThatThrownBy(() -> { + importEntities(importRequest); + }).hasMessageContaining("default device profile is present"); + + importRequest.getImportSettings().setFindExistingByName(true); + importEntities(importRequest); + checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); } + @Test + public void testExportImportDashboard_betweenTenants_doNotUpdateReferencesToOtherEntities() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "Customer 1"); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1"); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(customer.getId(), dashboard.getId())); + exportRequest.setExportSettings(new EntityExportSettings()); + List> exportDataList = exportEntities(exportRequest); + + logInAsTenantAdmin2(); + Map> importResults = importEntities(exportDataList).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); + + Customer importedCustomer = (Customer) importResults.get(EntityType.CUSTOMER).getSavedEntity(); + Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - private & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, EntityImportResult importResult) { - E importedEntity = importResult.getSavedEntity(); + logInAsTenantAdmin1(); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("aaa", new TextNode("bbb"))); + dashboard = dashboardService.saveDashboard(dashboard); + dashboard = dashboardService.unassignDashboardFromCustomer(tenantId1, dashboard.getId(), customer.getId()); + EntityExportData updatedDashboardExportData = exportSingleEntity(dashboard.getId()); + assertThat(updatedDashboardExportData.getEntity().getAssignedCustomers()).isNullOrEmpty(); - assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); - assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + logInAsTenantAdmin2(); + ImportRequest importRequest = new ImportRequest(); + importRequest.setExportDataList(List.of(updatedDashboardExportData)); + importRequest.setImportSettings(EntityImportSettings.builder() + .updateReferencesToOtherEntities(false) + .build()); + importedDashboard = (Dashboard) importEntities(importRequest).get(0).getSavedEntity(); + + assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } - assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); - boolean sameTenant = tenantId1.equals(tenantId2); - if (!sameTenant) { - assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); - } else { - assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); - assertThat(importResult.getOldEntity()).isEqualTo(initialEntity); + @Test + public void testExportRequests() throws Exception { + logInAsTenantAdmin1(); + + Device device = createDevice(tenantId1, null, null, "Device 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + Customer customer = createCustomer(tenantId1, "Customer 1"); + + Map> entities = Map.of( + EntityType.DEVICE, device, EntityType.DEVICE_PROFILE, deviceProfile, + EntityType.RULE_CHAIN, ruleChain, EntityType.ASSET, asset, + EntityType.DASHBOARD, dashboard, EntityType.CUSTOMER, customer + ); + + for (ExportableEntity entity : entities.values()) { + testEntityTypeExportRequest(entity); + testCustomEntityFilterExportRequest(entity); } } + private void testEntityTypeExportRequest(ExportableEntity entity) throws Exception { + EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setPageSize(10); + exportRequest.setEntityType(entity.getId().getEntityType()); + + List> exportDataList = exportEntities(exportRequest); + assertThat(exportDataList).size().isNotZero(); + assertThat(exportDataList).anySatisfy(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(entity); + }); + } + + private void testCustomEntityFilterExportRequest(ExportableEntity entity) throws Exception { + CustomEntityFilterExportRequest exportRequest = new CustomEntityFilterExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setPageSize(10); - private void logInAsTenantAdmin1() throws Exception { - login(tenantAdmin1.getEmail(), "12345678"); + EntityListFilter filter = new EntityListFilter(); + filter.setEntityType(entity.getId().getEntityType()); + filter.setEntityList(List.of(entity.getId().toString())); + exportRequest.setFilter(filter); + + List> exportDataList = exportEntities(exportRequest); + assertThat(exportDataList).hasOnlyOneElementSatisfying(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(entity); + }); } - private void logInAsTenantAdmin2() throws Exception { - login(tenantAdmin2.getEmail(), "12345678"); + + @Test + public void testExportImportCustomerEntities_betweenTenants() throws Exception { + logInAsTenantAdmin1(); + Customer customer = createCustomer(tenantId1, "Customer 1"); + + Device tenantDevice = createDevice(tenantId1, null, null, "Tenant device 1"); + Device customerDevice = createDevice(tenantId1, customer.getId(), null, "Customer device 1"); + Asset tenantAsset = createAsset(tenantId1, null, "A", "Tenant asset 1"); + Asset customerAsset = createAsset(tenantId1, customer.getId(), "A", "Customer asset 1"); + + List exportRequests = new ArrayList<>(); + + for (EntityType entityType : Set.of(EntityType.DEVICE, EntityType.ASSET)) { + EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + exportRequest.setExportSettings(new EntityExportSettings()); + exportRequest.setPageSize(10); + exportRequest.setEntityType(entityType); + exportRequest.setCustomerId(customer.getId()); + exportRequests.add(exportRequest); + } + + List> exportDataList = exportEntities(exportRequests); + assertThat(exportDataList).size().isEqualTo(2); + assertThat(exportDataList).anySatisfy(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(customerDevice); + }); + assertThat(exportDataList).anySatisfy(exportData -> { + assertThat(exportData.getEntity()).isEqualTo(customerAsset); + }); + } + + + @Test + public void testEntityEventsOnImport() throws Exception { + logInAsTenantAdmin1(); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + + EntityListExportRequest exportRequest = new EntityListExportRequest(); + exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), deviceProfile.getId())); + exportRequest.setExportSettings(new EntityExportSettings()); + + Map entitiesExportData = exportEntities(exportRequest).stream() + .collect(Collectors.toMap(EntityExportData::getEntityType, r -> r)); + + logInAsTenantAdmin2(); + + Customer importedCustomer = (Customer) importEntity(entitiesExportData.get(EntityType.CUSTOMER)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.ADDED), isNull()); + importEntity(entitiesExportData.get(EntityType.CUSTOMER)); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + Asset importedAsset = (Asset) importEntity(entitiesExportData.get(EntityType.ASSET)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.ADDED), isNull()); + importEntity(entitiesExportData.get(EntityType.ASSET)); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.UPDATED), isNull()); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + RuleChain importedRuleChain = (RuleChain) importEntity(entitiesExportData.get(EntityType.RULE_CHAIN)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); + + Dashboard importedDashboard = (Dashboard) importEntity(entitiesExportData.get(EntityType.DASHBOARD)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), + any(), eq(ActionType.ADDED), isNull()); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(entitiesExportData.get(EntityType.DEVICE_PROFILE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); + verify(clusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); + verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); + + ((DeviceExportData)entitiesExportData.get(EntityType.DEVICE)).getCredentials().setCredentialsId("abc"); + Device importedDevice = (Device) importEntity(entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), + any(), eq(ActionType.ADDED), isNull()); + verify(clusterService).onDeviceUpdated(eq(importedDevice), isNull()); + importEntity(entitiesExportData.get(EntityType.DEVICE)); + verify(clusterService).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index 50569b7a02..4da7de93ee 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -26,4 +26,7 @@ public interface ExportableEntity extends HasId, HasName I getExternalId(); void setExternalId(I externalId); + long getCreatedTime(); + void setCreatedTime(long createdTime); + } From 119430ff9a483d12f96f410d8c93dad1ffd11d87 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 18 Apr 2022 14:32:43 +0300 Subject: [PATCH 022/156] Minor improvements --- .../sync/DefaultEntitiesExportImportService.java | 12 ++++++++++-- .../sync/importing/impl/BaseEntityImportService.java | 2 +- .../sync/importing/impl/DashboardImportService.java | 10 ++++++++++ .../sync/importing/impl/RuleChainImportService.java | 10 ++++++++++ .../server/dao/dashboard/DashboardService.java | 5 +++++ .../server/dao/rule/RuleChainService.java | 3 +++ .../server/common/data/ExportableEntity.java | 1 - .../thingsboard/server/dao/ExportableEntityDao.java | 2 +- .../server/dao/dashboard/DashboardDao.java | 6 ++++++ .../server/dao/dashboard/DashboardServiceImpl.java | 7 +++++++ .../server/dao/rule/BaseRuleChainService.java | 7 ++++++- .../server/dao/sql/asset/JpaAssetDao.java | 2 +- .../server/dao/sql/customer/JpaCustomerDao.java | 2 +- .../dao/sql/dashboard/DashboardRepository.java | 3 ++- .../server/dao/sql/dashboard/JpaDashboardDao.java | 5 +++-- .../server/dao/sql/device/JpaDeviceDao.java | 2 +- .../server/dao/sql/device/JpaDeviceProfileDao.java | 2 +- .../server/dao/sql/rule/JpaRuleChainDao.java | 5 ----- .../server/dao/sql/rule/RuleChainRepository.java | 2 -- 19 files changed, 68 insertions(+), 20 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index cfb895c36a..1db59d68f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -83,7 +83,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + fixOrder(exportDataList); List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { @@ -101,6 +101,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } + private void fixOrder(List> exportDataList) { + exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + } + private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); @@ -137,7 +141,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - return dao.findFirstByTenantIdAndName(tenantId.getId(), name); + try { + return dao.findByTenantIdAndName(tenantId.getId(), name); + } catch (UnsupportedOperationException e) { + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 97547868d2..c7aaf9c198 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -158,7 +158,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) .or(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index fda6904421..c2a3e4aea7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.Collections; import java.util.HashSet; @@ -58,6 +59,15 @@ public class DashboardImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { Optional.ofNullable(dashboard.getConfiguration()) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 5fef00cbed..dcc3b6ca92 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.Collections; import java.util.Optional; @@ -50,6 +51,15 @@ public class RuleChainImportService extends BaseEntityImportService findDashboardsByTenantIdAndEdgeId(TenantId tenantId, EdgeId edgeId, PageLink pageLink); DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name); + + List findTenantDashboardsByTitle(TenantId tenantId, String title); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 4eed652bb7..9069b75710 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -32,6 +32,7 @@ import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.data.rule.RuleNode; +import java.util.Collection; import java.util.List; /** @@ -65,6 +66,8 @@ public interface RuleChainService { PageData findTenantRuleChainsByType(TenantId tenantId, RuleChainType type, PageLink pageLink); + Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name); + void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId); void deleteRuleChainsByTenantId(TenantId tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java index 4da7de93ee..c6d13013b1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -20,7 +20,6 @@ import org.thingsboard.server.common.data.id.HasId; public interface ExportableEntity extends HasId, HasName { - I getId(); void setId(I id); I getExternalId(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index 467389d9d0..cd534f24e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -23,6 +23,6 @@ public interface ExportableEntityDao> extends Dao< T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); - T findFirstByTenantIdAndName(UUID tenantId, String name); + default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index 90353f49b3..87804fab80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -21,6 +21,9 @@ import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; +import java.util.List; +import java.util.UUID; + /** * The Interface DashboardDao. */ @@ -33,4 +36,7 @@ public interface DashboardDao extends Dao, TenantEntityDao, Exportabl * @return saved dashboard object */ Dashboard save(TenantId tenantId, Dashboard dashboard); + + List findByTenantIdAndTitle(UUID tenantId, String title); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index ee5c44e20f..1e82d3f27c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -40,6 +40,8 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import java.util.List; + import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -279,6 +281,11 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb return dashboardInfoDao.findFirstByTenantIdAndName(tenantId.getId(), name); } + @Override + public List findTenantDashboardsByTitle(TenantId tenantId, String title) { + return dashboardDao.findByTenantIdAndTitle(tenantId.getId(), title); + } + private PaginatedRemover tenantDashboardsRemover = new PaginatedRemover() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index fd84c12a1b..8920cc0c1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -391,6 +391,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return ruleChainDao.findRuleChainsByTenantIdAndType(tenantId.getId(), type, pageLink); } + @Override + public Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name) { + return ruleChainDao.findByTenantIdAndTypeAndName(tenantId, type, name); + } + @Override @Transactional public void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId) { @@ -457,7 +462,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC ruleChain.setRoot(false); if (overwrite) { - Collection existingRuleChains = ruleChainDao.findByTenantIdAndTypeAndName(tenantId, + Collection existingRuleChains = findTenantRuleChainsByTypeAndName(tenantId, Optional.ofNullable(ruleChain.getType()).orElse(RuleChainType.CORE), ruleChain.getName()); Optional existingRuleChain = existingRuleChains.stream().findFirst(); if (existingRuleChain.isPresent()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index 5988c2d2db..bcfdaae965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -215,7 +215,7 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im } @Override - public Asset findFirstByTenantIdAndName(UUID tenantId, String name) { + public Asset findByTenantIdAndName(UUID tenantId, String name) { return findAssetsByTenantIdAndName(tenantId, name).orElse(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index fa09304427..c6ba310793 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -76,7 +76,7 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantIdAndTitle(UUID tenantId, String title); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 771de3c813..6ff9fc6bfb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; +import java.util.List; import java.util.UUID; /** @@ -58,8 +59,8 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantIdAndTitle(UUID tenantId, String title) { + return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index d2247f5e82..93cd857660 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -309,7 +309,7 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao } @Override - public Device findFirstByTenantIdAndName(UUID tenantId, String name) { + public Device findByTenantIdAndName(UUID tenantId, String name) { return findDeviceByTenantIdAndName(tenantId, name).orElse(null); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index b21a07442f..cae94cfb73 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -117,7 +117,7 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); - RuleChainEntity findFirstByTenantIdAndName(UUID tenantId, String name); - } From eacc0e6bbebd8f31dc30f9a8c129cf9b4d1f3946 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 18 Apr 2022 16:19:06 +0300 Subject: [PATCH 023/156] Remove import option whether to update references to other entities; minor improvements --- .../importing/data/EntityImportSettings.java | 2 - .../importing/impl/AssetImportService.java | 6 +-- .../impl/BaseEntityImportService.java | 43 +++-------------- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 23 +++++---- .../importing/impl/DeviceImportService.java | 12 ++--- .../impl/DeviceProfileImportService.java | 12 ++--- .../impl/RuleChainImportService.java | 8 ++-- ...aseEntitiesExportImportControllerTest.java | 4 +- ...EntitiesExportImportControllerSqlTest.java | 47 +------------------ 10 files changed, 42 insertions(+), 119 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index 3b457e1f5a..d4b0c46f31 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -30,6 +30,4 @@ public class EntityImportSettings { private boolean importInboundRelations; private boolean importOutboundRelations; private boolean removeExistingRelations; - - private boolean updateReferencesToOtherEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java index 02eaae27b5..ae54a988f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java @@ -36,13 +36,13 @@ public class AssetImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData exportData, IdProvider idProvider) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index c7aaf9c198..82df075238 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -40,13 +40,9 @@ import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.function.BiConsumer; -import java.util.function.Function; import java.util.stream.Collectors; public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @@ -68,7 +64,7 @@ public abstract class BaseEntityImportService importResult, D exportData, - NewIdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { + IdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { E savedEntity = importResult.getSavedEntity(); E oldEntity = importResult.getOldEntity(); @@ -179,37 +175,10 @@ public abstract class BaseEntityImportService ID get(Function idExtractor) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { - return getInternalId(idExtractor.apply(this.entity)); - } else { - return idExtractor.apply(existingEntity); - } - } - - public ID getInternal(ID externalId) { - return getInternalId(externalId); - } - - public Set get(Function> listExtractor, Function idGetter, BiConsumer idSetter) { - if (existingEntity == null || importSettings.isUpdateReferencesToOtherEntities()) { - return Optional.ofNullable(listExtractor.apply(entity)).orElse(Collections.emptySet()).stream() - .peek(t -> { - idSetter.accept(t, getInternalId(idGetter.apply(t))); - }) - .collect(Collectors.toSet()); - } else { - return listExtractor.apply(existingEntity); - } - } - private ID getInternalId(ID externalId) { + public ID getInternalId(ID externalId) { if (externalId == null || externalId.isNullUid()) return null; HasId entity = findInternalEntity(user.getTenantId(), externalId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java index b03c0ab335..1a79fe666b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java @@ -36,12 +36,12 @@ public class CustomerImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData exportData, IdProvider idProvider) { return customerService.saveCustomer(customer); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index c2a3e4aea7..e7c5ed0ea3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -55,7 +55,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider) { Optional.ofNullable(dashboard.getConfiguration()) .flatMap(configuration -> Optional.ofNullable(configuration.get("entityAliases"))) .filter(JsonNode::isObject) @@ -84,15 +84,17 @@ public class DashboardImportService extends BaseEntityImportService { String uuid = matchResult.group(); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); - return idProvider.getInternal(entityId).toString(); + return idProvider.getInternalId(entityId).toString(); }); ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); }); })); - // TODO [viacheslav]: improve the code below + Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() + .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) + .collect(Collectors.toSet()); + if (dashboard.getId() == null) { - Set assignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId); dashboard.setAssignedCustomers(null); dashboard = dashboardService.saveDashboard(dashboard); for (ShortCustomerInfo customerInfo : assignedCustomers) { @@ -101,20 +103,21 @@ public class DashboardImportService extends BaseEntityImportService existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); - Set newAssignedCustomers = idProvider.get(Dashboard::getAssignedCustomers, ShortCustomerInfo::getCustomerId, ShortCustomerInfo::setCustomerId).stream() - .map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set newAssignedCustomers = assignedCustomers.stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); Set toUnassign = new HashSet<>(existingAssignedCustomers); toUnassign.removeAll(newAssignedCustomers); for (CustomerId customerId : toUnassign) { - dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId); + assignedCustomers = dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); } + Set toAssign = new HashSet<>(newAssignedCustomers); toAssign.removeAll(existingAssignedCustomers); for (CustomerId customerId : toAssign) { - dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + assignedCustomers = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); } - dashboard.setAssignedCustomers(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()); + + dashboard.setAssignedCustomers(assignedCustomers); dashboard = dashboardService.saveDashboard(dashboard); } return dashboard; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java index 504bd1a828..091b6eb856 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java @@ -35,16 +35,16 @@ public class DeviceImportService extends BaseEntityImportService exportData, NewIdProvider idProvider) { - deviceProfile.setDefaultRuleChainId(idProvider.get(DeviceProfile::getDefaultRuleChainId)); - deviceProfile.setDefaultDashboardId(idProvider.get(DeviceProfile::getDefaultDashboardId)); - deviceProfile.setFirmwareId(idProvider.get(DeviceProfile::getFirmwareId)); - deviceProfile.setSoftwareId(idProvider.get(DeviceProfile::getSoftwareId)); + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { + deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); + deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); + deviceProfile.setSoftwareId(idProvider.getInternalId(deviceProfile.getSoftwareId())); return deviceProfileService.saveDeviceProfile(deviceProfile); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index dcc3b6ca92..1e4ed958a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -47,7 +47,7 @@ public class RuleChainImportService extends BaseEntityImportService { @@ -73,14 +73,14 @@ public class RuleChainImportService extends BaseEntityImportService { ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.getInternal(new RuleChainId(otherRuleChainUuid)).toString() + idProvider.getInternalId(new RuleChainId(otherRuleChainUuid)).toString() )); ruleNode.setConfiguration(ruleNodeConfig); }); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { - ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternal(ruleChainConnectionInfo.getTargetRuleChainId())); + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId())); }); ruleChain.setFirstRuleNodeId(null); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index a8a5bca534..b5c5a5aab8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -360,9 +360,7 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected List> importEntities(List> exportDataList) throws Exception { ImportRequest importRequest = new ImportRequest(); - importRequest.setImportSettings(EntityImportSettings.builder() - .updateReferencesToOtherEntities(true) - .build()); + importRequest.setImportSettings(new EntityImportSettings()); importRequest.setExportDataList(exportDataList); return importEntities(importRequest); } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 920d91e93d..8a0c958b2c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -376,9 +376,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin2(); ImportRequest importRequest = new ImportRequest(); - importRequest.setImportSettings(EntityImportSettings.builder() - .updateReferencesToOtherEntities(true) - .build()); + importRequest.setImportSettings(new EntityImportSettings()); importRequest.setExportDataList(exportDataList); Map> importResults = importEntities(importRequest).stream() .collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); @@ -583,7 +581,6 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp importRequest.setExportDataList(exportDataList); importRequest.setImportSettings(EntityImportSettings.builder() .findExistingByName(false) - .updateReferencesToOtherEntities(true) .build()); assertThatThrownBy(() -> { importEntities(importRequest); @@ -594,48 +591,6 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); } - @Test - public void testExportImportDashboard_betweenTenants_doNotUpdateReferencesToOtherEntities() throws Exception { - logInAsTenantAdmin1(); - Customer customer = createCustomer(tenantId1, "Customer 1"); - Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1"); - - EntityListExportRequest exportRequest = new EntityListExportRequest(); - exportRequest.setEntitiesIds(List.of(customer.getId(), dashboard.getId())); - exportRequest.setExportSettings(new EntityExportSettings()); - List> exportDataList = exportEntities(exportRequest); - - logInAsTenantAdmin2(); - Map> importResults = importEntities(exportDataList).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); - - Customer importedCustomer = (Customer) importResults.get(EntityType.CUSTOMER).getSavedEntity(); - Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).getSavedEntity(); - assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { - assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); - }); - assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - - logInAsTenantAdmin1(); - dashboard.setConfiguration(JacksonUtil.newObjectNode().set("aaa", new TextNode("bbb"))); - dashboard = dashboardService.saveDashboard(dashboard); - dashboard = dashboardService.unassignDashboardFromCustomer(tenantId1, dashboard.getId(), customer.getId()); - EntityExportData updatedDashboardExportData = exportSingleEntity(dashboard.getId()); - assertThat(updatedDashboardExportData.getEntity().getAssignedCustomers()).isNullOrEmpty(); - - logInAsTenantAdmin2(); - ImportRequest importRequest = new ImportRequest(); - importRequest.setExportDataList(List.of(updatedDashboardExportData)); - importRequest.setImportSettings(EntityImportSettings.builder() - .updateReferencesToOtherEntities(false) - .build()); - importedDashboard = (Dashboard) importEntities(importRequest).get(0).getSavedEntity(); - - assertThat(importedDashboard.getConfiguration()).isEqualTo(dashboard.getConfiguration()); - assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { - assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); - }); - } - @Test public void testExportRequests() throws Exception { From a5cdbca690b431bca9c7ad99a585840f50edac99 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 13:37:45 +0300 Subject: [PATCH 024/156] Relations import: log entity action --- .../impl/BaseEntityImportService.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 82df075238..5802e140f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -91,11 +91,16 @@ public abstract class BaseEntityImportService importResult, D exportData, IdProvider idProvider, EntityImportSettings importSettings) throws ThingsboardException { E savedEntity = importResult.getSavedEntity(); E oldEntity = importResult.getOldEntity(); + importResult.addSendEventsCallback(() -> { + onEntitySaved(user, savedEntity, oldEntity); + }); + importResult.addSaveReferencesCallback(() -> { List newRelations = new LinkedList<>(); @@ -107,7 +112,7 @@ public abstract class BaseEntityImportService { + entityActionService.logEntityAction(user, relation.getFrom(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityActionService.logEntityAction(user, relation.getTo(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + }); } }); + } + private void deleteRelation(SecurityUser user, EntityRelation relation, EntityImportResult importResult) { + relationService.deleteRelation(user.getTenantId(), relation); importResult.addSendEventsCallback(() -> { - onEntitySaved(user, savedEntity, oldEntity); + entityActionService.logEntityAction(user, relation.getFrom(), null, null, + ActionType.RELATION_DELETED, null, relation); + entityActionService.logEntityAction(user, relation.getTo(), null, null, + ActionType.RELATION_DELETED, null, relation); }); } + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), From 832f32fbe1e5cfb28c2a47f30e9210fd62b9ece7 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 14:34:55 +0300 Subject: [PATCH 025/156] Improvements for relations import --- .../sync/exporting/data/EntityExportData.java | 3 +- .../impl/DefaultEntityExportService.java | 22 ++--- .../importing/data/EntityImportSettings.java | 5 +- .../impl/BaseEntityImportService.java | 89 +++++++++---------- ...EntitiesExportImportControllerSqlTest.java | 27 +++--- 5 files changed, 68 insertions(+), 78 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java index 4ab6dd57b1..728d8212c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java @@ -43,7 +43,6 @@ public class EntityExportData> { private E entity; private EntityType entityType; - private List inboundRelations; - private List outboundRelations; + private List relations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java index 4fe09e9322..672bc50109 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import java.util.ArrayList; import java.util.List; @Service @@ -63,24 +64,25 @@ public class DefaultEntityExportService relations = null; if (exportSettings.isExportInboundRelations()) { List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (inboundRelations != null) { - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); } - exportData.setInboundRelations(inboundRelations); + relations = new ArrayList<>(inboundRelations); } if (exportSettings.isExportOutboundRelations()) { List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - if (outboundRelations != null) { - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } - exportData.setOutboundRelations(outboundRelations); + if (relations == null) { + relations = new ArrayList<>(); + } + relations.addAll(outboundRelations); } + exportData.setRelations(relations); } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java index d4b0c46f31..a5de7d8744 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java @@ -26,8 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityImportSettings { private boolean findExistingByName; - - private boolean importInboundRelations; - private boolean importOutboundRelations; - private boolean removeExistingRelations; + private boolean updateRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index 5802e140f3..1edd6a1ec9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.importing.impl; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -40,10 +39,9 @@ import org.thingsboard.server.service.sync.importing.EntityImportService; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @@ -102,47 +100,53 @@ public abstract class BaseEntityImportService { - List newRelations = new LinkedList<>(); - - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - newRelations.addAll(exportData.getInboundRelations().stream() - .peek(relation -> relation.setTo(savedEntity.getId())) - .collect(Collectors.toList())); - - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getFrom(), Operation.WRITE); - deleteRelation(user, existingRelation, importResult); - } - } - } - if (importSettings.isImportOutboundRelations() && CollectionUtils.isNotEmpty(exportData.getOutboundRelations())) { - newRelations.addAll(exportData.getOutboundRelations().stream() - .peek(relation -> relation.setFrom(savedEntity.getId())) - .collect(Collectors.toList())); - - if (importSettings.isRemoveExistingRelations() && oldEntity != null) { - for (EntityRelation existingRelation : relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)) { - exportableEntitiesService.checkPermission(user, existingRelation.getTo(), Operation.WRITE); - deleteRelation(user, existingRelation, importResult); - } - } + List relations = exportData.getRelations(); + if (relations == null || !importSettings.isUpdateRelations()) { + return; } + relations = new ArrayList<>(relations); - for (EntityRelation relation : newRelations) { - HasId otherEntity = null; + for (EntityRelation relation : relations) { if (!relation.getTo().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getTo()); - relation.setTo(otherEntity.getId()); + HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); + exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); + relation.setTo(to.getId()); } - if (!relation.getFrom().equals(savedEntity.getId())) { - otherEntity = findInternalEntity(user.getTenantId(), relation.getFrom()); - relation.setFrom(otherEntity.getId()); + if (!relation.getFrom().equals(savedEntity.getId())){ + HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); + exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); + relation.setFrom(from.getId()); } - if (otherEntity != null) { - exportableEntitiesService.checkPermission(user, otherEntity, otherEntity.getId().getEntityType(), Operation.WRITE); + } + + if (oldEntity != null) { + List existingRelations = new ArrayList<>(); + existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + + for (EntityRelation existingRelation : existingRelations) { + if (!relations.contains(existingRelation)) { + EntityId otherEntity = null; + if (!existingRelation.getTo().equals(savedEntity.getId())) { + otherEntity = existingRelation.getTo(); + } else if (!existingRelation.getFrom().equals(savedEntity.getId())){ + otherEntity = existingRelation.getFrom(); + } + if (otherEntity != null) { + exportableEntitiesService.checkPermission(user, otherEntity, Operation.WRITE); + } + relationService.deleteRelation(user.getTenantId(), existingRelation); + importResult.addSendEventsCallback(() -> { + entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + entityActionService.logEntityAction(user, existingRelation.getTo(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + }); + } } + } + for (EntityRelation relation : relations) { relationService.saveRelation(user.getTenantId(), relation); importResult.addSendEventsCallback(() -> { entityActionService.logEntityAction(user, relation.getFrom(), null, null, @@ -154,17 +158,6 @@ public abstract class BaseEntityImportService importResult) { - relationService.deleteRelation(user.getTenantId(), relation); - importResult.addSendEventsCallback(() -> { - entityActionService.logEntityAction(user, relation.getFrom(), null, null, - ActionType.RELATION_DELETED, null, relation); - entityActionService.logEntityAction(user, relation.getTo(), null, null, - ActionType.RELATION_DELETED, null, relation); - }); - } - - protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 8a0c958b2c..8d1857f133 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -311,7 +311,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Dashboard importedDashboard = (Dashboard) importResults.get(EntityType.DASHBOARD).get(0).getSavedEntity(); Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() - .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) .map(JsonNode::asText).collect(Collectors.toSet()); assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); @@ -427,8 +427,8 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp List> exportDataList = exportEntities(exportRequest); EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); - assertThat(deviceExportData.getInboundRelations()).size().isOne(); - assertThat(deviceExportData.getInboundRelations().get(0)).matches(entityRelation -> { + assertThat(deviceExportData.getRelations()).size().isOne(); + assertThat(deviceExportData.getRelations().get(0)).matches(entityRelation -> { return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); }); ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); @@ -439,7 +439,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(exportDataList); importRequest.setImportSettings(EntityImportSettings.builder() - .importInboundRelations(true) + .updateRelations(true) .build()); Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); @@ -473,7 +473,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp .build()); List> exportDataList = exportEntities(exportRequest); - assertThat(exportDataList).allMatch(exportData -> exportData.getInboundRelations().size() + exportData.getOutboundRelations().size() == 1); + assertThat(exportDataList).allMatch(exportData -> exportData.getRelations().size() == 1); EntityExportData deviceExportData = exportDataList.stream().filter(exportData -> exportData.getEntityType() == EntityType.DEVICE).findFirst().orElse(null); ((DeviceExportData) deviceExportData).getCredentials().setCredentialsId("ab"); @@ -484,8 +484,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(exportDataList); importRequest.setImportSettings(EntityImportSettings.builder() - .importInboundRelations(true) - .importOutboundRelations(true) + .updateRelations(true) .build()); Map> importResults = importEntities(importRequest).stream().collect(Collectors.toMap(EntityImportResult::getEntityType, r -> r)); @@ -515,7 +514,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp .exportOutboundRelations(true) .build()); EntityExportData assetExportData = (EntityExportData) exportEntities(exportRequest).get(0); - assertThat(assetExportData.getOutboundRelations()).size().isOne(); + assertThat(assetExportData.getRelations()).size().isOne(); Device device2 = createDevice(tenantId1, null, null, "Device 2"); EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); @@ -523,13 +522,14 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(List.of(assetExportData)); importRequest.setImportSettings(EntityImportSettings.builder() - .importOutboundRelations(true) + .updateRelations(true) .build()); importEntities(importRequest); List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); - assertThat(relations).contains(relation1, relation2); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); } @Test @@ -546,7 +546,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp .exportInboundRelations(true) .build()); EntityExportData deviceExportData = exportEntities(exportRequest).get(0); - assertThat(deviceExportData.getInboundRelations()).size().isOne(); + assertThat(deviceExportData.getRelations()).size().isOne(); Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); @@ -554,8 +554,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp ImportRequest importRequest = new ImportRequest(); importRequest.setExportDataList(List.of(deviceExportData)); importRequest.setImportSettings(EntityImportSettings.builder() - .importInboundRelations(true) - .removeExistingRelations(true) + .updateRelations(true) .build()); importEntities(importRequest); @@ -730,7 +729,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp verify(clusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); - ((DeviceExportData)entitiesExportData.get(EntityType.DEVICE)).getCredentials().setCredentialsId("abc"); + ((DeviceExportData) entitiesExportData.get(EntityType.DEVICE)).getCredentials().setCredentialsId("abc"); Device importedDevice = (Device) importEntity(entitiesExportData.get(EntityType.DEVICE)).getSavedEntity(); verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), any(), eq(ActionType.ADDED), isNull()); From 457d2340a905ec5d98346ac56618cdf5d9f46044 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 16:23:54 +0300 Subject: [PATCH 026/156] Refactor relations export --- .../data/request/EntityExportSettings.java | 3 +-- .../impl/DefaultEntityExportService.java | 16 +++++++--------- .../EntitiesExportImportControllerSqlTest.java | 10 ++++------ 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java index ddf2e43bcf..5454105ee5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java @@ -25,6 +25,5 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @Builder public class EntityExportSettings { - private boolean exportInboundRelations; - private boolean exportOutboundRelations; + private boolean exportRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java index 672bc50109..a051f0e59f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java @@ -64,25 +64,23 @@ public class DefaultEntityExportService relations = null; - if (exportSettings.isExportInboundRelations()) { + if (exportSettings.isExportRelations()) { + List relations = new ArrayList<>(); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); for (EntityRelation relation : inboundRelations) { exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); } - relations = new ArrayList<>(inboundRelations); - } - if (exportSettings.isExportOutboundRelations()) { + relations.addAll(inboundRelations); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); for (EntityRelation relation : outboundRelations) { exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } - if (relations == null) { - relations = new ArrayList<>(); - } relations.addAll(outboundRelations); + + exportData.setRelations(relations); } - exportData.setRelations(relations); } protected D newExportData() { diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 8d1857f133..44d8f5f461 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -421,8 +421,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityListExportRequest exportRequest = new EntityListExportRequest(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportInboundRelations(true) - .exportOutboundRelations(false) + .exportRelations(true) .build()); List> exportDataList = exportEntities(exportRequest); @@ -468,8 +467,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp EntityListExportRequest exportRequest = new EntityListExportRequest(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportInboundRelations(true) - .exportOutboundRelations(true) + .exportRelations(true) .build()); List> exportDataList = exportEntities(exportRequest); @@ -511,7 +509,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); exportRequest.setEntityId(asset.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportOutboundRelations(true) + .exportRelations(true) .build()); EntityExportData assetExportData = (EntityExportData) exportEntities(exportRequest).get(0); assertThat(assetExportData.getRelations()).size().isOne(); @@ -543,7 +541,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); exportRequest.setEntityId(device.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() - .exportInboundRelations(true) + .exportRelations(true) .build()); EntityExportData deviceExportData = exportEntities(exportRequest).get(0); assertThat(deviceExportData.getRelations()).size().isOne(); From 428ff2851dd3fbf189b58b64baea369b13b8297f Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 21 Apr 2022 18:25:56 +0300 Subject: [PATCH 027/156] Minor refactoring --- .../sync/DefaultEntitiesExportImportService.java | 2 +- .../sync/exporting/impl/DeviceExportService.java | 2 +- .../sync/importing/impl/BaseEntityImportService.java | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 1db59d68f1..44a9396fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -153,7 +153,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { if (entity instanceof HasTenantId) { accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); - } else if (entity != null) { + } else { accessControlService.checkPermission(user, Resource.of(entityType), operation); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java index da1e4f3a96..21df8beda2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java @@ -36,7 +36,7 @@ public class DeviceExportService extends BaseEntityExportService { - List relations = exportData.getRelations(); - if (relations == null || !importSettings.isUpdateRelations()) { + if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) { return; } - relations = new ArrayList<>(relations); + + List relations = new ArrayList<>(exportData.getRelations()); for (EntityRelation relation : relations) { if (!relation.getTo().equals(savedEntity.getId())) { @@ -112,7 +112,7 @@ public abstract class BaseEntityImportService from = findInternalEntity(user.getTenantId(), relation.getFrom()); exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); @@ -129,7 +129,7 @@ public abstract class BaseEntityImportService Date: Thu, 21 Apr 2022 19:13:58 +0300 Subject: [PATCH 028/156] Refactor export/import api --- .../EntitiesExportImportController.java | 104 ++------- .../DefaultEntitiesExportImportService.java | 88 +------- .../DefaultExportableEntitiesService.java | 206 ++++++++++++++++++ .../exporting/ExportableEntitiesService.java | 5 + .../sync/exporting/data/DeviceExportData.java | 2 + .../exporting/data/RuleChainExportData.java | 2 + 6 files changed, 239 insertions(+), 168 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 4a1badd38e..3dff560ac1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -17,47 +17,25 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.dao.DataAccessException; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.query.EntityData; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityFilter; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; -import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; - -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @RestController @RequestMapping("/api/entities") @@ -67,7 +45,8 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; - private final EntityService entityService; + private final ExportableEntitiesService exportableEntitiesService; + @PostMapping("/export") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -76,6 +55,7 @@ public class EntitiesExportImportController extends BaseController { try { return exportEntitiesByRequest(user, exportRequest); } catch (Exception e) { + log.warn("Failed to export entities for request {}", exportRequest, e); throw handleException(e); } } @@ -85,84 +65,29 @@ public class EntitiesExportImportController extends BaseController { public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List> exportDataList = new ArrayList<>(); + List> result = new ArrayList<>(); for (ExportRequest exportRequest : exportRequests) { - exportDataList.addAll(exportEntitiesByRequest(user, exportRequest)); + List> exportDataList = exportEntitiesByRequest(user, exportRequest); + result.addAll(exportDataList); } - return exportDataList; + return result; } catch (Exception e) { + log.warn("Failed to export entities for requests {}", exportRequests, e); throw handleException(e); } } - - private List> exportEntitiesByRequest(SecurityUser user, ExportRequest request) throws ThingsboardException { - List entitiesIds = findEntitiesForRequest(user, request); + private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { + List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); List> exportDataList = new ArrayList<>(); - for (EntityId entityId : entitiesIds) { - exportDataList.add(exportImportService.exportEntity(user, entityId, request.getExportSettings())); + for (EntityId entityId : entities) { + EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportRequest.getExportSettings()); + exportDataList.add(exportData); } return exportDataList; } - private List findEntitiesForRequest(SecurityUser user, ExportRequest request) { - switch (request.getType()) { - case SINGLE_ENTITY: { - return List.of(((SingleEntityExportRequest) request).getEntityId()); - } - case ENTITY_LIST: { - return ((EntityListExportRequest) request).getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(exportRequest.getEntityType()); - - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); - return findEntitiesByFilter(user.getTenantId(), customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); - } - case CUSTOM_ENTITY_FILTER: { - CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; - EntityFilter filter = exportRequest.getFilter(); - - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); - return findEntitiesByFilter(user.getTenantId(), customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); - } - case CUSTOM_ENTITY_QUERY: { - CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; - EntityDataQuery query = exportRequest.getQuery(); - - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(emptyId(EntityType.CUSTOMER)); - return findEntitiesByQuery(user.getTenantId(), customerId, query); - } - default: - throw new IllegalArgumentException("Export request is not supported"); - } - } - - private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - - EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - return findEntitiesByQuery(tenantId, customerId, query); - } - - private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { - try { - return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() - .map(EntityData::getEntityId) - .collect(Collectors.toList()); - } catch (DataAccessException e) { - log.error("Failed to find entity data by query: {}", e.getMessage()); - throw new IllegalArgumentException("Entity filter cannot be processed"); - } - } - @PostMapping("/import") public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { @@ -183,6 +108,7 @@ public class EntitiesExportImportController extends BaseController { return importResults; } catch (Exception e) { + log.warn("Failed to import entities for request {}", importRequest, e); throw handleException(e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 44a9396fab..75c4ff9e12 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -16,26 +16,18 @@ package org.thingsboard.server.service.sync; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.HasId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.Dao; -import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.AccessControlService; -import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; @@ -57,13 +49,11 @@ import java.util.stream.Collectors; @Service @TbCoreComponent @RequiredArgsConstructor -public class DefaultEntitiesExportImportService implements EntitiesExportImportService, ExportableEntitiesService { +@Slf4j +public class DefaultEntitiesExportImportService implements EntitiesExportImportService { private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final Map> daos = new HashMap<>(); - - private final AccessControlService accessControlService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -117,75 +107,24 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - @Override - public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { - EntityType entityType = externalId.getEntityType(); - if (SUPPORTED_ENTITY_TYPES.contains(entityType)) { - ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - return dao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); - } else { - return null; - } - } - - @Override - public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { - Dao dao = (Dao) getDao(id.getEntityType()); - E entity = dao.findById(tenantId, id.getId()); - if (entity instanceof HasTenantId && !((HasTenantId) entity).getTenantId().equals(tenantId)) { - return null; - } - return entity; - } - - @Override - public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { - ExportableEntityDao dao = (ExportableEntityDao) getDao(entityType); - try { - return dao.findByTenantIdAndName(tenantId.getId(), name); - } catch (UnsupportedOperationException e) { - return null; - } - } - - - @Override - public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { - if (entity instanceof HasTenantId) { - accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); - } else { - accessControlService.checkPermission(user, Resource.of(entityType), operation); - } - } - - @Override - public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { - HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); - checkPermission(user, entity, entityId.getEntityType(), operation); - } - - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { - if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + EntityExportService exportService = exportServices.get(entityType); + if (exportService == null) { throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); } - return (EntityExportService) exportServices.get(entityType); + return (EntityExportService) exportService; } @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { - if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + EntityImportService importService = importServices.get(entityType); + if (importService == null) { throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); } - return (EntityImportService) importServices.get(entityType); + return (EntityImportService) importService; } - private Dao getDao(EntityType entityType) { - return daos.get(entityType); - } - - @Autowired private void setExportServices(DefaultEntityExportService defaultExportService, Collection> exportServices) { @@ -208,13 +147,4 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS }); } - @Autowired - private void setDaos(Collection> daos) { - daos.forEach(dao -> { - if (dao.getEntityType() != null) { - this.daos.put(dao.getEntityType(), dao); - } - }); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java new file mode 100644 index 0000000000..c2c99c63f5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -0,0 +1,206 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.exporting; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataAccessException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.AccessControlService; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultExportableEntitiesService implements ExportableEntitiesService { + + private final Map> daos = new HashMap<>(); + + private final EntityService entityService; + private final AccessControlService accessControlService; + + + @Override + public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { + EntityType entityType = externalId.getEntityType(); + Dao dao = getDao(entityType); + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + return exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } else { + return null; + } + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + + E entity = dao.findById(tenantId, id.getId()); + if (((HasTenantId) entity).getTenantId().equals(tenantId)) { + return entity; + } + return null; + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { + Dao dao = getDao(entityType); + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + try { + return exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); + } catch (UnsupportedOperationException ignored) { + } + } + return null; + } + + + @Override + public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { + switch (request.getType()) { + case SINGLE_ENTITY: { + return List.of(((SingleEntityExportRequest) request).getEntityId()); + } + case ENTITY_LIST: { + return ((EntityListExportRequest) request).getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(exportRequest.getEntityType()); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case CUSTOM_ENTITY_FILTER: { + CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; + EntityFilter filter = exportRequest.getFilter(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + } + case CUSTOM_ENTITY_QUERY: { + CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; + EntityDataQuery query = exportRequest.getQuery(); + + CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + return findEntitiesByQuery(tenantId, customerId, query); + } + default: { + throw new IllegalArgumentException("Export request is not supported"); + } + } + } + + private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + + EntityDataQuery query = new EntityDataQuery(filter, pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + return findEntitiesByQuery(tenantId, customerId, query); + } + + private List findEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityDataQuery query) { + try { + return entityService.findEntityDataByQuery(tenantId, customerId, query).getData().stream() + .map(EntityData::getEntityId) + .collect(Collectors.toList()); + } catch (DataAccessException e) { + log.error("Failed to find entity data by query: {}", e.getMessage()); + throw new IllegalArgumentException("Entity filter cannot be processed"); + } + } + + + @Override + public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { + if (entity instanceof HasTenantId) { + accessControlService.checkPermission(user, Resource.of(entityType), operation, entity.getId(), (HasTenantId) entity); + } else { + accessControlService.checkPermission(user, Resource.of(entityType), operation); + } + } + + @Override + public void checkPermission(SecurityUser user, EntityId entityId, Operation operation) throws ThingsboardException { + HasId entity = findEntityByTenantIdAndId(user.getTenantId(), entityId); + checkPermission(user, entity, entityId.getEntityType(), operation); + } + + + @SuppressWarnings("unchecked") + private Dao getDao(EntityType entityType) { + return (Dao) daos.get(entityType); + } + + @Autowired + private void setDaos(Collection> daos) { + daos.forEach(dao -> { + if (dao.getEntityType() != null) { + this.daos.put(dao.getEntityType(), dao); + } + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java index 5cf31042e0..9021a03e85 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java @@ -23,6 +23,9 @@ import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; + +import java.util.List; public interface ExportableEntitiesService { @@ -32,6 +35,8 @@ public interface ExportableEntitiesService { , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); + List findEntitiesForRequest(TenantId tenantId, ExportRequest request); + void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java index 30d84c0a44..56633f9c91 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java @@ -17,10 +17,12 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.ToString; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.security.DeviceCredentials; @EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) @Data public class DeviceExportData extends EntityExportData { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java index 025459f27e..2e05e2f1f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java @@ -17,10 +17,12 @@ package org.thingsboard.server.service.sync.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.ToString; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) @Data public class RuleChainExportData extends EntityExportData { From 7e2910ed9549bf5fe6973530293fc46bd751c2d2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 25 Apr 2022 14:28:54 +0300 Subject: [PATCH 029/156] Minor improvements --- .../EntitiesExportImportController.java | 20 +++++++++- .../DefaultExportableEntitiesService.java | 39 +++++++++++++------ .../CustomEntityFilterExportRequest.java | 5 ++- .../CustomEntityQueryExportRequest.java | 5 ++- .../data/request/EntityTypeExportRequest.java | 5 ++- 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index 3dff560ac1..afe2a4ac1f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -29,8 +29,10 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.EntitiesExportImportService; import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; import java.util.ArrayList; @@ -79,10 +81,16 @@ public class EntitiesExportImportController extends BaseController { private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); + EntityExportSettings exportSettings = exportRequest.getExportSettings(); + if (exportSettings == null) { + exportSettings = EntityExportSettings.builder() + .exportRelations(false) + .build(); + } List> exportDataList = new ArrayList<>(); for (EntityId entityId : entities) { - EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportRequest.getExportSettings()); + EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportSettings); exportDataList.add(exportData); } return exportDataList; @@ -93,7 +101,15 @@ public class EntitiesExportImportController extends BaseController { public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importRequest.getImportSettings()); + EntityImportSettings importSettings = importRequest.getImportSettings(); + if (importSettings == null) { + importSettings = EntityImportSettings.builder() + .findExistingByName(false) + .updateRelations(false) + .build(); + } + + List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importSettings); importResults.stream() .map(EntityImportResult::getSendEventsCallback) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java index c2c99c63f5..a616dec6c4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.sync.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; @@ -56,7 +57,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; @@ -78,12 +78,17 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi EntityType entityType = externalId.getEntityType(); Dao dao = getDao(entityType); + E entity = null; + if (dao instanceof ExportableEntityDao) { ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; - return exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); - } else { + entity = exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } + if (entity == null || !belongsToTenant(entity, tenantId)) { return null; } + + return entity; } @Override @@ -92,24 +97,36 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi Dao dao = getDao(entityType); E entity = dao.findById(tenantId, id.getId()); - if (((HasTenantId) entity).getTenantId().equals(tenantId)) { - return entity; + + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; } - return null; + + return entity; } @Override public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { Dao dao = getDao(entityType); + E entity = null; + if (dao instanceof ExportableEntityDao) { ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; try { - return exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); + entity = exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); } catch (UnsupportedOperationException ignored) { } } - return null; + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + + return entity; + } + + private boolean belongsToTenant(HasId entity, TenantId tenantId) { + return tenantId.equals(((HasTenantId) entity).getTenantId()); } @@ -127,21 +144,21 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); entityTypeFilter.setEntityType(exportRequest.getEntityType()); - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); } case CUSTOM_ENTITY_FILTER: { CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; EntityFilter filter = exportRequest.getFilter(); - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); } case CUSTOM_ENTITY_QUERY: { CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; EntityDataQuery query = exportRequest.getQuery(); - CustomerId customerId = Optional.ofNullable(exportRequest.getCustomerId()).orElse(new CustomerId(EntityId.NULL_UUID)); + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); return findEntitiesByQuery(tenantId, customerId, query); } default: { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java index 02cbca98d6..d71ddc8720 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java @@ -17,9 +17,10 @@ package org.thingsboard.server.service.sync.exporting.data.request; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.query.EntityFilter; +import java.util.UUID; + @EqualsAndHashCode(callSuper = true) @Data public class CustomEntityFilterExportRequest extends ExportRequest { @@ -27,7 +28,7 @@ public class CustomEntityFilterExportRequest extends ExportRequest { private EntityFilter filter; private int page; private int pageSize; - private CustomerId customerId; + private UUID customerId; @Override public ExportRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java index 76d258e891..946676cf65 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java @@ -17,15 +17,16 @@ package org.thingsboard.server.service.sync.exporting.data.request; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.query.EntityDataQuery; +import java.util.UUID; + @EqualsAndHashCode(callSuper = true) @Data public class CustomEntityQueryExportRequest extends ExportRequest { private EntityDataQuery query; - private CustomerId customerId; + private UUID customerId; @Override public ExportRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java index 82cb023258..fd2f659b70 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java @@ -18,7 +18,8 @@ package org.thingsboard.server.service.sync.exporting.data.request; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.CustomerId; + +import java.util.UUID; @EqualsAndHashCode(callSuper = true) @Data @@ -27,7 +28,7 @@ public class EntityTypeExportRequest extends ExportRequest { private EntityType entityType; private int page; private int pageSize; - private CustomerId customerId; + private UUID customerId; @Override public ExportRequestType getType() { From 5736e52d3e38a5ffaff6c7f0a6a91ebdba22c788 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 25 Apr 2022 14:54:53 +0300 Subject: [PATCH 030/156] Fix EntitiesExportImportControllerSqlTest --- .../server/controller/EntitiesExportImportController.java | 1 + .../controller/sql/EntitiesExportImportControllerSqlTest.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index afe2a4ac1f..a6ad9f1ca6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -81,6 +81,7 @@ public class EntitiesExportImportController extends BaseController { private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); + EntityExportSettings exportSettings = exportRequest.getExportSettings(); if (exportSettings == null) { exportSettings = EntityExportSettings.builder() diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index 44d8f5f461..bd237e39d0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -659,7 +659,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entityType); - exportRequest.setCustomerId(customer.getId()); + exportRequest.setCustomerId(customer.getUuidId()); exportRequests.add(exportRequest); } From ba1ea3521034b2eb9a6cb24c5165bf88396480be Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 26 Apr 2022 12:08:05 +0300 Subject: [PATCH 031/156] Swagger docs for export-import controller --- .../EntitiesExportImportController.java | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index a6ad9f1ca6..c91fd0c292 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiOperation; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.security.access.prepost.PreAuthorize; @@ -39,6 +40,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; + @RestController @RequestMapping("/api/entities") @TbCoreComponent @@ -50,6 +53,42 @@ public class EntitiesExportImportController extends BaseController { private final ExportableEntitiesService exportableEntitiesService; + @ApiOperation(value = "Export entities by request", notes = "" + + "Takes export request and returns list of export data for each entity found by request. " + + "Supported entity types for export, hence for import, are **DEVICE**, **DEVICE_PROFILE**, **ASSET**, " + + "**CUSTOMER**, **RULE_CHAIN** and **DASHBOARD**." + NEW_LINE + + "For each type of export request, you can set some export settings: \n" + + "- **exportRelations** - whether to export inbound and outbound relations for an entity " + + "(only relations of type group COMMON can be exported)" + NEW_LINE + + "Supported export requests:\n" + + "- **SINGLE_ENTITY**:" + NEW_LINE + + " To export a single entity by id. Example:" + NEW_LINE + + "```\n{\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + + "- **ENTITY_LIST**:" + NEW_LINE + + " To export a list of entities by their ids. Example:" + NEW_LINE + + "```\n{\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"ASSET\",\n \"id\": \"2f0a3bd0-989d-11ec-93b5-6de6c2b68078\"\n }\n ],\n \"exportSettings\": {\n \"exportRelations\": true\n }\n}\n```" + NEW_LINE + + "- **ENTITY_TYPE**:" + NEW_LINE + + " To export entities of specified entity type. You need to specify page size, " + + "and may specify page index and customer id (to limit the list of entities to the ones owned by a customer). " + + "Entities are ordered by created time descendingly. Example:" + NEW_LINE + + "```\n{\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n}\n```" + NEW_LINE + + "- **CUSTOM_ENTITY_FILTER**:" + NEW_LINE + + " To export entities by custom entity filter. The order used is the same as for ENTITY_TYPE export request. Example:" + NEW_LINE + + "```\n{\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"Thermostats\",\n \"deviceNameFilter\": \"\"\n },\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + + "- **CUSTOM_ENTITY_QUERY**:" + NEW_LINE + + " To export entities by custom entity query. Example: " + NEW_LINE + + "```\n{\n \"type\": \"CUSTOM_ENTITY_QUERY\",\n \"query\": {\n \"entityFilter\": {\n \"type\": \"entityType\",\n \"entityType\": \"DEVICE\"\n },\n \"pageLink\": {\n \"page\": 0,\n \"pageSize\": 200,\n \"textSearch\": \"THB_\",\n \"sortOrder\": {\n \"key\": {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n },\n \"direction\": \"DESC\"\n }\n },\n \"entityFields\": [\n {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n }\n ],\n \"latestValues\": [\n {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n }\n ],\n \"keyFilters\": [\n {\n \"key\": {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n },\n \"valueType\": \"NUMERIC\",\n \"predicate\": {\n \"type\": \"NUMERIC\",\n \"operation\": \"GREATER\",\n \"value\": {\n \"defaultValue\": 0\n }\n }\n }\n ]\n },\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + + "Mostly, export data of an entity contains the whole entity itself and its relations " + + "(if option to export relations was enabled):" + NEW_LINE + + "```\n[\n {\n \"entityType\": \"ASSET\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1648204424029,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": { ... },\n \"customerId\": { ... },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n ...\n },\n \"relations\": [\n {\n \"from\": {\n \"entityType\": \"ASSET\",\n \"id\": ...\n },\n \"to\": {\n \"entityType\": \"DEVICE\",\n \"id\": ...\n },\n \"type\": \"Contains\",\n \"typeGroup\": \"COMMON\",\n \"additionalInfo\": {\n \"a\": \"b\"\n }\n }\n ]\n }\n]\n```" + NEW_LINE + + "For devices, export data will additionally contain device's credentials; for rule chains - its metadata:" + NEW_LINE + + "```\n[\n {\n \"entityType\": \"DEVICE\",\n \"entity\": { ... },\n \"credentials\": {\n \"id\": { ... },\n \"createdTime\": 1648829321209,\n \"deviceId\": { ... },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"5cZEDo45KGW7JgVNv4Ko\",\n \"credentialsValue\": null\n }\n }\n]\n```" + NEW_LINE + + "```\n[\n {\n \"entityType\": \"RULE_CHAIN\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1646056614257,\n \"additionalInfo\": null,\n \"tenantId\": { ... },\n \"name\": \"Rule Chain 2\",\n \"type\": \"CORE\",\n \"firstRuleNodeId\": { ... },\n \"root\": false,\n ...\n },\n \"metaData\": {\n \"ruleChainId\": { ... },\n \"firstNodeIndex\": 7,\n \"nodes\": [ ... ],\n \"connections\": [ ... ],\n \"ruleChainConnections\": null\n }\n }\n]\n```" + NEW_LINE + + "Returned export data is to be used later for import request." + NEW_LINE + + "If any entity found by request is of unsupported type - an error will be returned.\n" + + "Also, if a user does not have a READ permission for an entity (or, if relations are exported, for a bounded entity), " + + "access will be denied." + + ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) @PostMapping("/export") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { @@ -62,6 +101,12 @@ public class EntitiesExportImportController extends BaseController { } } + @ApiOperation(value = "Export entities by multiple requests", notes = "" + + "The API behaviour is the same as for exporting entities by single request, " + + "except that this method takes an array of export requests as a request body." + NEW_LINE + + "Example:" + NEW_LINE + + "```\n[\n {\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"5f9eda10-b442-11ec-bbf5-adec34031568\"\n }\n },\n {\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"thermostat\",\n \"deviceNameFilter\": \"\"\n },\n \"pageSize\": 1000\n },\n {\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"pageSize\": 1000,\n \"exportSettings\": {\n \"exportRelations\": true\n }\n },\n {\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"2ef13590-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"e7311ec0-b442-11ec-bbf5-adec34031568\"\n }\n ]\n }\n]\n```" + + ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) @PostMapping(value = "/export", params = {"multiple"}) @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { @@ -98,6 +143,34 @@ public class EntitiesExportImportController extends BaseController { } + @ApiOperation(value = "Import entities by request", notes = "" + + "Takes import request and returns the list of import results. " + + "Import request must contain the list of export data and might contain import settings. " + NEW_LINE + + "The method creates an entity if it is new in the scope of a tenant, or otherwise updates an existing one. " + + "On entity import request, we first try to find an entity within a tenant that has externalId equal " + + "to the id in the export data. If the platform fails to do that, we then search for an entity with " + + "regular (internal) id like the one in the export data (this is useful in case we are exporting and " + + "importing entities within the same tenant). Then, if we still haven't found any entity, if findExistingByName " + + "option of the EntityImportSettings is enabled, we will search for the one by its name (this is also useful " + + "for avoiding conflicts with default device profile or Root Rule Chain when importing all entities from another " + + "tenant). After, if the exported entity is new for this tenant, we simply save it with external id " + + "from the export data, and also create relations if any (if updateRelations option is enabled). " + + "Otherwise, we will reset all fields of the existing entity to the ones from the export data and save it, " + + "and also will update the list of relations (remove the ones that aren't present in the export data, " + + "and update or create others), if updateRelations option is enabled." + NEW_LINE + + "If an entity contains references to some other entities, like device references certain device profile, " + + "we will find this other entity within the tenant by this principle: look for an entity with " + + "such external id, or otherwise, internal id. This requires referenced entities to be imported " + + "before the referencing entity (if we are importing to another tenant). So, when receiving the list " + + "of entities' export data for import, we first try to fix the data order to import 'standalone' entities first." + NEW_LINE + + "As for relations importing, they are processed after all entities in the import batch are already saved, " + + "and the internal id of a bounded entity is found with the regular principle." + NEW_LINE + + "Import of all entities and their relations from the import request is processed in the single transaction, " + + "and so everything will be rolled back if the platform fails to e.g. find internal entity by external id. \n" + + "Example of import request:\n" + + "```\n{\n \"importSettings\": {\n \"findExistingByName\": false,\n \"updateRelations\": false\n },\n \"exportDataList\": [\n {\n \"entityType\": \"DEVICE_PROFILE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n \"createdTime\": 1649096026765,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"name\": \"Profile 1\",\n ...\n }\n },\n {\n \"entityType\": \"DEVICE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154276962,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"customerId\": {\n \"entityType\": \"CUSTOMER\",\n \"id\": \"13814000-1dd2-11b2-8080-808080808080\"\n },\n \"name\": \"Device 1\",\n \"type\": \"Profile 1\",\n \"label\": \"v1.0\",\n \"deviceProfileId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n ...\n },\n \"credentials\": {\n \"id\": {\n \"id\": \"981e0360-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154277014,\n \"deviceId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"sGExNdnl71uKmkNvtNdp\",\n \"credentialsValue\": null\n }\n }\n ]\n}\n```" + NEW_LINE + + "The response contains a list of EntityImportResult which has values of savedEntity and oldEntity:\n" + + "```\n[\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v2.0\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"6b03ab20-989e-11ec-b446-89df822d7fa2\"\n }\n },\n \"oldEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v1.0\",\n ...\n \"externalId\": null\n },\n \"entityType\": \"ASSET\"\n },\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"387213f0-b4ea-11ec-abfc-6dcc6508d0b5\"\n },\n \"createdTime\": 1649167860399,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 2\",\n \"type\": \"B\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"0b9ea0d0-ac27-11ec-a2a6-89d15eae3b21\"\n }\n },\n \"oldEntity\": null,\n \"entityType\": \"ASSET\"\n }\n]\n```") @PostMapping("/import") public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { SecurityUser user = getCurrentUser(); From a1d62373c9784b32c454b45a0fe3b5a409533814 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 26 Apr 2022 12:08:28 +0300 Subject: [PATCH 032/156] Transaction timeout for finding entities by request and importing entities batch --- .../server/service/sync/DefaultEntitiesExportImportService.java | 2 +- .../sync/exporting/DefaultExportableEntitiesService.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java index 75c4ff9e12..55384b21db 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java @@ -70,7 +70,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } - @Transactional(rollbackFor = Exception.class) + @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { fixOrder(exportDataList); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java index a616dec6c4..bc8f406c54 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; @@ -130,6 +131,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } + @Transactional(readOnly = true, timeout = 40) @Override public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { switch (request.getType()) { From 335b4ef4652a07d9ddf032941c5c4f7833624ceb Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Sat, 14 May 2022 17:23:34 +0300 Subject: [PATCH 033/156] Improvements for import of rule chains and dashboards --- .../DefaultExportableEntitiesService.java | 3 ++ .../impl/BaseEntityImportService.java | 24 +++++++++++++ .../impl/DashboardImportService.java | 35 +++++------------- .../impl/RuleChainImportService.java | 29 +++++++-------- .../thingsboard/server/utils/RegexUtils.java | 36 +++++++++++++++++++ 5 files changed, 85 insertions(+), 42 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/utils/RegexUtils.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java index bc8f406c54..ce6924de93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java @@ -96,6 +96,9 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } E entity = dao.findById(tenantId, id.getId()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java index e96359a746..28095a7b85 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java @@ -20,11 +20,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -42,6 +44,7 @@ import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @@ -201,6 +204,27 @@ public abstract class BaseEntityImportService getInternalIdByUuid(UUID externalUuid) { + for (EntityType entityType : EntityType.values()) { + EntityId externalId; + try { + externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); + } catch (Exception e) { + continue; + } + + EntityId internalId = null; + try { + internalId = getInternalId(externalId); + } catch (Exception ignored) {} + + if (internalId != null) { + return Optional.of(internalId); + } + } + return Optional.empty(); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java index e7c5ed0ea3..77f32bf94c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.importing.impl; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -27,22 +26,19 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.dao.dashboard.DashboardService; -import org.thingsboard.server.dao.sql.query.DefaultEntityQueryRepository; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; import java.util.HashSet; import java.util.Optional; import java.util.Set; -import java.util.regex.Pattern; +import java.util.UUID; import java.util.stream.Collectors; @Service @@ -52,7 +48,6 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - Optional.ofNullable(dashboard.getConfiguration()) - .flatMap(configuration -> Optional.ofNullable(configuration.get("entityAliases"))) - .filter(JsonNode::isObject) - .ifPresent(entityAliases -> entityAliases.forEach(entityAlias -> { - Optional.ofNullable(entityAlias.get("filter")) - .filter(JsonNode::isObject) - .ifPresent(filter -> { - EntityFilter entityFilter = JacksonUtil.treeToValue(filter, EntityFilter.class); - EntityType entityType = DefaultEntityQueryRepository.resolveEntityType(entityFilter); - - String filterJson = filter.toString(); - String newFilterJson = UUID_PATTERN.matcher(filterJson).replaceAll(matchResult -> { - String uuid = matchResult.group(); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, uuid); - return idProvider.getInternalId(entityId).toString(); - }); - ((ObjectNode) entityAlias).set("filter", JacksonUtil.toJsonNode(newFilterJson)); - }); - })); + JsonNode configuration = dashboard.getConfiguration(); + String newConfigurationJson = RegexUtils.replace(configuration.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + .map(entityId -> entityId.getId().toString()).orElse(uuid); + }); + configuration = JacksonUtil.toJsonNode(newConfigurationJson); + dashboard.setConfiguration(configuration); Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java index 1e4ed958a8..09a13d5f14 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java @@ -16,10 +16,9 @@ package org.thingsboard.server.service.sync.importing.impl; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; @@ -29,11 +28,13 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; import java.util.Optional; @@ -67,16 +68,15 @@ public class RuleChainImportService extends BaseEntityImportService { ruleNode.setId(null); ruleNode.setRuleChainId(null); + JsonNode ruleNodeConfig = ruleNode.getConfiguration(); - Optional.ofNullable(ruleNodeConfig) - .flatMap(config -> Optional.ofNullable(config.get("ruleChainId")).filter(JsonNode::isTextual)) - .map(JsonNode::asText).map(UUID::fromString) - .ifPresent(otherRuleChainUuid -> { - ((ObjectNode) ruleNodeConfig).set("ruleChainId", new TextNode( - idProvider.getInternalId(new RuleChainId(otherRuleChainUuid)).toString() - )); - ruleNode.setConfiguration(ruleNodeConfig); - }); + String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> { + return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + .map(entityId -> entityId.getId().toString()) + .orElse(uuid); + }); + ruleNodeConfig = JacksonUtil.toJsonNode(newRuleNodeConfigJson); + ruleNode.setConfiguration(ruleNodeConfig); }); Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) .forEach(ruleChainConnectionInfo -> { @@ -84,13 +84,10 @@ public class RuleChainImportService extends BaseEntityImportService replacer) { + return pattern.matcher(s).replaceAll(matchResult -> { + return replacer.apply(matchResult.group()); + }); + } + +} From f82be0153b4e77aba4d686face2b0861ff2af607 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Sat, 2 Apr 2022 19:42:48 +0300 Subject: [PATCH 034/156] Entities VC with Git - initial implementation --- application/pom.xml | 4 + .../EntitiesVersionControlController.java | 118 ++++++++ .../DefaultEntitiesVersionControlService.java | 274 ++++++++++++++++++ .../vcs/EntitiesVersionControlService.java | 50 ++++ .../data/EntitiesVersionControlSettings.java | 28 ++ .../service/sync/vcs/data/EntityVersion.java | 29 ++ .../service/sync/vcs/data/GitSettings.java | 32 ++ .../server/utils/git/Repository.java | 256 ++++++++++++++++ .../server/utils/git/data/Branch.java | 23 ++ .../server/utils/git/data/Commit.java | 25 ++ .../server/utils/git/data/Diff.java | 25 ++ pom.xml | 6 + 12 files changed, 870 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/Repository.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java create mode 100644 application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java diff --git a/application/pom.xml b/application/pom.xml index 1b89752925..2bfab2c858 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -337,6 +337,10 @@ Java-WebSocket test + + org.eclipse.jgit + org.eclipse.jgit + diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java new file mode 100644 index 0000000000..10928f1b51 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; +import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vcs.data.EntityVersion; + +import java.util.List; +import java.util.Set; +import java.util.UUID; + +@RestController +@RequestMapping("/api/entities/vc") +@RequiredArgsConstructor +public class EntitiesVersionControlController extends BaseController { + + private final DefaultEntitiesVersionControlService versionControlService; + + + + @PostMapping("/version/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, + @RequestBody String versionName) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); + } + + @GetMapping("/version/{entityType}/{entityId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listEntityVersions(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); + } + + + + @GetMapping("/entity/{entityType}/{entityId}/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @PathVariable String versionId) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId); + } + + @PostMapping("/entity/{entityType}/{entityId}/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, + @PathVariable("entityId") UUID entityUuid, + @PathVariable String versionId) throws Exception { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId); + } + + + + @GetMapping("/branches") + public Set getAllowedBranches() throws ThingsboardException { + return versionControlService.getAllowedBranches(getTenantId()); + } + + + @PostMapping("/settings") + @PreAuthorize("hasAuthority('SYS_ADMIN')") + public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { + versionControlService.saveSettings(settings); + } + + @GetMapping("/settings") + @PreAuthorize("hasAuthority('SYS_ADMIN')") + public EntitiesVersionControlSettings getSettings() { + return versionControlService.getSettings(); + } + + + + @PostMapping("/repository/reset") + @PreAuthorize("hasAuthority('SYS_ADMIN')") + public void resetLocalRepository() throws Exception { + versionControlService.resetRepository(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..0d78496f16 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java @@ -0,0 +1,274 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.EntityExportSettings; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.importing.EntityImportSettings; +import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vcs.data.EntityVersion; +import org.thingsboard.server.service.sync.vcs.data.GitSettings; +import org.thingsboard.server.utils.git.Repository; +import org.thingsboard.server.utils.git.data.Commit; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + // TODO [viacheslav]: start up only on one of the cores + + private final TenantService tenantService; + private final EntitiesExportImportService exportImportService; + private final AdminSettingsService adminSettingsService; + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private static final String SETTINGS_KEY = "vc"; + + private Repository repository; + private final ReentrantLock fetchLock = new ReentrantLock(); + private final Lock writeLock = new ReentrantLock(); + + @AfterStartUp + public void init() throws Exception { + try { + EntitiesVersionControlSettings settings = getSettings(); + if (settings != null && settings.getGitSettings() != null) { + this.repository = initRepository(settings.getGitSettings()); + } + } catch (Exception e) { + log.error("Failed to initialize entities version control service", e); + } + } + + + @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) + public void fetch() throws Exception { + if (repository == null) return; + + if (fetchLock.tryLock()) { + try { + log.info("Fetching remote repository"); + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + } + + + @Override + public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { + return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); + } + + @Override + public EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception { + checkRepository(); + checkBranch(tenantId, branch); + + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportInboundRelations(false) + .exportOutboundRelations(false) + .build(); + List>> entityDataList = entitiesIds.stream() + .map(entityId -> { + return exportImportService.exportEntity(tenantId, entityId, exportSettings); + }) + .collect(Collectors.toList()); + + if (fetchLock.tryLock()) { + try { + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + + writeLock.lock(); + try { + if (repository.listBranches().contains(branch)) { + repository.checkout(branch); + repository.merge(branch); + } else { + repository.createAndCheckoutOrphanBranch(branch); + } + + for (EntityExportData> entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), + entityDataJson, StandardCharsets.UTF_8); + } + + Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); + repository.push(); + return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + } finally { + writeLock.unlock(); + } + } + + + + @Override + public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream() + .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) + .collect(Collectors.toList()); + } + + @Override + public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream() + .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) + .collect(Collectors.toList()); + } + + + + @Override + public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception { + checkRepository(); + // FIXME [viacheslav]: validate access + + String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); + return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + } + + @Override + public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception { + EntityExportData entityData = getEntityAtVersion(tenantId, entityId, versionId); + return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() + .importInboundRelations(false) + .importOutboundRelations(false) + .updateReferencesToOtherEntities(true) + .build()); + } + + + + private String getRelativePathForEntity(EntityId entityId) { + return getRelativePathForEntityType(entityId.getEntityType()) + + "/" + entityId.getId() + ".json"; + } + + private String getRelativePathForEntityType(EntityType entityType) { + return entityType.name().toLowerCase(); + } + + + private void checkBranch(TenantId tenantId, String branch) { + if (!getAllowedBranches(tenantId).contains(branch)) { + throw new IllegalArgumentException("Tenant does not have access to this branch"); + } + } + + public Set getAllowedBranches(TenantId tenantId) { + return Optional.ofNullable(getSettings()) + .flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) + .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) + .orElse(Collections.emptySet()); + } + + @Override + public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { + this.repository = initRepository(settings.getGitSettings()); + + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) + .orElseGet(() -> { + AdminSettings newSettings = new AdminSettings(); + newSettings.setKey(SETTINGS_KEY); + return newSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); + } + + @Override + public EntitiesVersionControlSettings getSettings() { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + .orElse(null); + } + + + + private void checkRepository() { + if (repository == null) { + throw new IllegalStateException("Repository is not initialized"); + } + } + + private static Repository initRepository(GitSettings gitSettings) throws Exception { + if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { + return Repository.open(gitSettings.getRepositoryDirectory(), + gitSettings.getUsername(), gitSettings.getPassword()); + } else { + Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); + return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), + gitSettings.getUsername(), gitSettings.getPassword()); + } + } + + public void resetRepository() throws Exception { + if (this.repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + this.repository = null; + } + EntitiesVersionControlSettings settings = getSettings(); + this.repository = initRepository(settings.getGitSettings()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java new file mode 100644 index 0000000000..a9a3c0e3e8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.importing.EntityImportResult; +import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vcs.data.EntityVersion; + +import java.util.List; + +public interface EntitiesVersionControlService { + + EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; + + EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception; + + + List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; + + List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; + + + , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + + , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + + + void saveSettings(EntitiesVersionControlSettings settings) throws Exception; + + EntitiesVersionControlSettings getSettings(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java new file mode 100644 index 0000000000..6a46606845 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs.data; + +import lombok.Data; + +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +@Data +public class EntitiesVersionControlSettings { + private Map> allowedBranches; + private GitSettings gitSettings; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java new file mode 100644 index 0000000000..42a2e2f555 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs.data; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class EntityVersion { + private String id; + private String name; + private String authorName; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java new file mode 100644 index 0000000000..0d97bed2d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vcs.data; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class GitSettings { + private String repositoryUri; + private String repositoryDirectory; + private String username; + private String password; +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java new file mode 100644 index 0000000000..8099be8259 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java @@ -0,0 +1,256 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git; + +import com.google.common.collect.Streams; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.utils.git.data.Commit; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class Repository { + + private final Git git; + private final CredentialsProvider credentialsProvider; + + @Getter + private final String directory; + + private Repository(Git git, CredentialsProvider credentialsProvider, String directory) { + this.git = git; + this.credentialsProvider = credentialsProvider; + this.directory = directory; + } + + public static Repository clone(String uri, String directory, + String username, String password) throws GitAPIException { + CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); + Git git = Git.cloneRepository() + .setURI(uri) + .setDirectory(new java.io.File(directory)) + .setNoCheckout(true) + .setCredentialsProvider(credentialsProvider) + .call(); + return new Repository(git, credentialsProvider, directory); + } + + public static Repository open(String directory, String username, String password) throws IOException { + Git git = Git.open(new java.io.File(directory)); + return new Repository(git, newCredentialsProvider(username, password), directory); + } + + + public void fetch() throws GitAPIException { + execute(git.fetch() + .setRemoveDeletedRefs(true)); + } + + + public List listBranches() throws GitAPIException { + return execute(git.branchList() + .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .filter(ref -> !ref.getName().equals(Constants.HEAD)) + .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) + .map(name -> StringUtils.removeStart(name, "origin/")) + .distinct().collect(Collectors.toList()); + } + + + public List listCommits(String branchName, int limit) throws IOException, GitAPIException { + return listCommits(branchName, null, limit); + } + + public List listCommits(String branchName, String path, int limit) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branchName); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + LogCommand command = git.log() + .add(branchId).setMaxCount(limit) + .setRevFilter(RevFilter.NO_MERGES); + if (StringUtils.isNotEmpty(path)) { + command.addPath(path); + } + return Streams.stream(execute(command)) + .map(this::toCommit) + .collect(Collectors.toList()); + } + + + public List listFilesAtCommit(Commit commit) throws IOException { + return listFilesAtCommit(commit, null); + } + + public List listFilesAtCommit(Commit commit, String path) throws IOException { + List files = new ArrayList<>(); + RevCommit revCommit = resolveCommit(commit.getId()); + try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { + treeWalk.reset(revCommit.getTree().getId()); + if (StringUtils.isNotEmpty(path)) { + treeWalk.setFilter(PathFilter.create(path)); + } + treeWalk.setRecursive(true); + while (treeWalk.next()) { + files.add(treeWalk.getPathString()); + } + } + return files; + } + + + public String getFileContentAtCommit(String file, String commitId) throws IOException { + RevCommit revCommit = resolveCommit(commitId); + try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { + if (treeWalk == null) { + throw new IllegalArgumentException("Not found"); + } + ObjectId blobId = treeWalk.getObjectId(0); + try (ObjectReader objectReader = git.getRepository().newObjectReader()) { + ObjectLoader objectLoader = objectReader.open(blobId); + byte[] bytes = objectLoader.getBytes(); + return new String(bytes, StandardCharsets.UTF_8); + } + } + } + + + public void checkout(String branchName) throws GitAPIException { + execute(git.checkout() + .setName(branchName)); + } + + public void merge(String branchName) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branchName); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + + public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { + execute(git.checkout() + .setOrphan(true) + .setName(name)); + Set uncommittedChanges = git.status().call().getUncommittedChanges(); + if (!uncommittedChanges.isEmpty()) { + RmCommand rm = git.rm(); + uncommittedChanges.forEach(rm::addFilepattern); + execute(rm); + } + execute(git.clean()); + } + + public void clean() throws GitAPIException { + execute(git.clean().setCleanDirectories(true)); + } + + public Commit commit(String message, String filePattern, String author) throws GitAPIException { + execute(git.add().addFilepattern(filePattern)); + RevCommit revCommit = execute(git.commit() + .setMessage(message) + .setAuthor(author, author)); + return toCommit(revCommit); + } + + + public void push() throws GitAPIException { + execute(git.push()); + } + + +// public List getCommitChanges(Commit commit) throws IOException, GitAPIException { +// RevCommit revCommit = resolveCommit(commit.getId()); +// if (revCommit.getParentCount() == 0) { +// return null; // just takes the first parent of a commit, but should find a parent in branch provided +// } +// return execute(git.diff() +// .setOldTree(prepareTreeParser(git.getRepository().parseCommit(revCommit.getParent(0)))) +// .setNewTree(prepareTreeParser(revCommit))).stream() +// .map(diffEntry -> new Diff(diffEntry.getChangeType().name(), diffEntry.getOldPath(), diffEntry.getNewPath())) +// .collect(Collectors.toList()); +// } +// +// +// private AbstractTreeIterator prepareTreeParser(RevCommit revCommit) throws IOException { +// // from the commit we can build the tree which allows us to construct the TreeParser +// //noinspection Duplicates +// org.eclipse.jgit.lib.Repository repository = git.getRepository(); +// try (RevWalk walk = new RevWalk(repository)) { +// RevTree tree = walk.parseTree(revCommit.getTree().getId()); +// +// CanonicalTreeParser treeParser = new CanonicalTreeParser(); +// try (ObjectReader reader = repository.newObjectReader()) { +// treeParser.reset(reader, tree.getId()); +// } +// +// walk.dispose(); +// +// return treeParser; +// } +// } + + private Commit toCommit(RevCommit revCommit) { + return new Commit(revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + } + + private RevCommit resolveCommit(String id) throws IOException { + return git.getRepository().parseCommit(resolve(id)); + } + + private ObjectId resolve(String rev) throws IOException { + return git.getRepository().resolve(rev); + } + + private , T> T execute(C command) throws GitAPIException { + if (command instanceof TransportCommand) { + ((TransportCommand) command).setCredentialsProvider(credentialsProvider); +// SshSessionFactory sshSessionFactory = SshSessionFactory.getInstance(); +// transportCommand.setTransportConfigCallback(transport -> { +// ((SshTransport) transport).setSshSessionFactory(sshSessionFactory); +// }); + } + return command.call(); + } + + private static CredentialsProvider newCredentialsProvider(String username, String password) { + return new UsernamePasswordCredentialsProvider(username, password); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java b/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java new file mode 100644 index 0000000000..a458f9f5aa --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git.data; + +import lombok.Data; + +@Data +public class Branch { + private final String shortName; +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java b/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java new file mode 100644 index 0000000000..7567dba1c5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git.data; + +import lombok.Data; + +@Data +public class Commit { + private final String id; + private final String message; + private final String authorName; +} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java b/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java new file mode 100644 index 0000000000..7572a003d2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.utils.git.data; + +import lombok.Data; + +@Data +public class Diff { + private final String type; + private final String oldPath; + private final String newPath; +} diff --git a/pom.xml b/pom.xml index 9713e86993..ea5df86409 100755 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 1.16.0 1.12 + 6.1.0.202203080745-r @@ -1875,6 +1876,11 @@ ${zeroturnaround.version} test + + org.eclipse.jgit + org.eclipse.jgit + ${jgit.version} + From b3dfed5badc997144dc69588aaef2e5ff44b0c79 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 4 Apr 2022 12:00:02 +0300 Subject: [PATCH 035/156] API for listing entities at version, loading all entities at version, listing all available versions --- .../EntitiesVersionControlController.java | 64 +++++- .../DefaultEntitiesVersionControlService.java | 184 +++++++++++++----- .../vcs/EntitiesVersionControlService.java | 11 +- .../server/utils/git/Repository.java | 8 +- 4 files changed, 207 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 10928f1b51..798556998c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vcs.data.EntityVersion; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; @RestController @RequestMapping("/api/entities/vc") @@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController { @PostMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, + @PathVariable("entityId") UUID id, @RequestParam String branch, @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); } + @PostMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, + @RequestParam UUID[] ids, + @RequestParam String branch, + @RequestBody String versionName) throws Exception { + List entitiesIds = Arrays.stream(ids) + .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) + .collect(Collectors.toList()); + return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); + } + + + @GetMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List listEntityVersions(@PathVariable EntityType entityType, @@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController { return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); } + @GetMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch) throws Exception { + return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); + } + + @GetMapping("/version") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listVersions(@RequestParam String branch) throws Exception { + return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); + } + + + + @GetMapping("/files/version/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listFilesAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); + } + @GetMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId); + return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); } @PostMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId); + EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); + onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + return result; + } + + @PostMapping("/entity/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> loadAllAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + SecurityUser user = getCurrentUser(); + List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); + resultList.forEach(result -> { + onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + }); + return resultList; } @GetMapping("/branches") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") public Set getAllowedBranches() throws ThingsboardException { return versionControlService.getAllowedBranches(getTenantId()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java index 0d78496f16..32da1309fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository; import org.thingsboard.server.utils.git.data.Commit; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -54,7 +57,9 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Service @@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private static final String SETTINGS_KEY = "vc"; private Repository repository; - private final ReentrantLock fetchLock = new ReentrantLock(); - private final Lock writeLock = new ReentrantLock(); + private final Lock fetchLock = new ReentrantLock(); + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); @AfterStartUp public void init() throws Exception { @@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - public void fetch() throws Exception { + private void fetch() throws Exception { if (repository == null) return; - - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); } @@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportOutboundRelations(false) .build(); List>> entityDataList = entitiesIds.stream() - .map(entityId -> { - return exportImportService.exportEntity(tenantId, entityId, exportSettings); - }) + .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) .collect(Collectors.toList()); - if (fetchLock.tryLock()) { - try { - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); - writeLock.lock(); + repositoryLock.writeLock().lock(); try { if (repository.listBranches().contains(branch)) { repository.checkout(branch); @@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); repository.push(); - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + return toVersion(commit); } finally { - writeLock.unlock(); + repositoryLock.writeLock().unlock(); } } @@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); } @Override public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); + return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); + } - return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + @Override + public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { + return listVersions(tenantId, branch, null, limit); + } + + private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, limit).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + + } finally { + repositoryLock.readLock().unlock(); + } } @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - checkRepository(); - // FIXME [viacheslav]: validate access + public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + return repository.listFilesAtCommit(versionId); + } finally { + repositoryLock.readLock().unlock(); + } + } - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + + + @Override + public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + + String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); + return parseEntityData(entityDataJson); + } finally { + repositoryLock.readLock().unlock(); + } } + @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, versionId); + public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() .importInboundRelations(false) .importOutboundRelations(false) @@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build()); } + @Override + public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() + .map(entityDataFilePath -> { + String entityDataJson; + try { + entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return parseEntityData(entityDataJson); + }) + .collect(Collectors.toList()); + + return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() + .importInboundRelations(false) + .importOutboundRelations(false) + .updateReferencesToOtherEntities(true) + .build()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + private void tryFetch() throws GitAPIException { + repositoryLock.readLock().lock(); + try { + if (fetchLock.tryLock()) { + try { + log.info("Fetching remote repository"); + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + } finally { + repositoryLock.readLock().unlock(); + } + } private String getRelativePathForEntity(EntityId entityId) { @@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void checkBranch(TenantId tenantId, String branch) { + // TODO [viacheslav]: all branches are available by default? if (!getAllowedBranches(tenantId).contains(branch)) { throw new IllegalArgumentException("Tenant does not have access to this branch"); } @@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .orElse(Collections.emptySet()); } + private EntityVersion toVersion(Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + } + + private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { + return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + } + + + @Override public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - this.repository = initRepository(settings.getGitSettings()); + repositoryLock.writeLock().lock(); + try { + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); + } AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) .orElseGet(() -> { @@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } - private void checkRepository() { if (repository == null) { throw new IllegalStateException("Repository is not initialized"); @@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } public void resetRepository() throws Exception { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; + repositoryLock.writeLock().lock(); + try { + if (this.repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + this.repository = null; + } + EntitiesVersionControlSettings settings = getSettings(); + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java index a9a3c0e3e8..e05ad17826 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java @@ -37,10 +37,17 @@ public interface EntitiesVersionControlService { List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; + List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception; - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; void saveSettings(EntitiesVersionControlSettings settings) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java index 8099be8259..23eb7d7db4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java @@ -113,13 +113,13 @@ public class Repository { } - public List listFilesAtCommit(Commit commit) throws IOException { - return listFilesAtCommit(commit, null); + public List listFilesAtCommit(String commitId) throws IOException { + return listFilesAtCommit(commitId, null); } - public List listFilesAtCommit(Commit commit, String path) throws IOException { + public List listFilesAtCommit(String commitId, String path) throws IOException { List files = new ArrayList<>(); - RevCommit revCommit = resolveCommit(commit.getId()); + RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { treeWalk.reset(revCommit.getTree().getId()); if (StringUtils.isNotEmpty(path)) { From 073875f406ed4437ec844a3a29a116effba5be44 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 28 Apr 2022 13:26:11 +0300 Subject: [PATCH 036/156] Entities VC improvements and refactoring --- .../EntitiesVersionControlController.java | 272 ++++++------ .../DefaultEntitiesVersionControlService.java | 395 ++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 64 +++ .../data/EntitiesVersionControlSettings.java} | 20 +- .../sync/{vcs => vc}/data/EntityVersion.java | 5 +- .../data/EntityVersionLoadResult.java} | 21 +- .../vc/data/EntityVersionLoadSettings.java} | 9 +- .../vc/data/EntityVersionSaveSettings.java} | 6 +- .../sync/vc/data/VersionedEntityInfo.java} | 11 +- .../DefaultEntitiesVersionControlService.java | 358 ---------------- .../vcs/EntitiesVersionControlService.java | 57 --- .../Repository.java => GitRepository.java} | 79 ++-- 12 files changed, 665 insertions(+), 632 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/GitSettings.java => vc/data/EntitiesVersionControlSettings.java} (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs => vc}/data/EntityVersion.java (90%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/EntitiesVersionControlSettings.java => vc/data/EntityVersionLoadResult.java} (57%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Commit.java => service/sync/vc/data/EntityVersionLoadSettings.java} (78%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Branch.java => service/sync/vc/data/EntityVersionSaveSettings.java} (83%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Diff.java => service/sync/vc/data/VersionedEntityInfo.java} (74%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/utils/{git/Repository.java => GitRepository.java} (82%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 798556998c..c39442341e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,159 +16,139 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @RestController @RequestMapping("/api/entities/vc") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { - private final DefaultEntitiesVersionControlService versionControlService; - - - - @PostMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID id, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); - return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); - } - - @PostMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, - @RequestParam UUID[] ids, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - List entitiesIds = Arrays.stream(ids) - .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) - .collect(Collectors.toList()); - return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); - } - - - - @GetMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityVersions(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityTypeVersions(@PathVariable EntityType entityType, - @RequestParam String branch) throws Exception { - return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listVersions(@RequestParam String branch) throws Exception { - return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); - } - - - - @GetMapping("/files/version/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listFilesAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); - } - - - - @GetMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); - } - - @PostMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); - onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - return result; - } - - @PostMapping("/entity/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> loadAllAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - SecurityUser user = getCurrentUser(); - List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); - resultList.forEach(result -> { - onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - }); - return resultList; - } - - - - @GetMapping("/branches") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public Set getAllowedBranches() throws ThingsboardException { - return versionControlService.getAllowedBranches(getTenantId()); - } - - - @PostMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { - versionControlService.saveSettings(settings); - } - - @GetMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public EntitiesVersionControlSettings getSettings() { - return versionControlService.getSettings(); - } - - - - @PostMapping("/repository/reset") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void resetLocalRepository() throws Exception { - versionControlService.resetRepository(); - } + private final EntitiesVersionControlService versionControlService; + + + // search request - export request with settings + +// +// @PostMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID id, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); +// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); +// } +// +// @PostMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, +// @RequestParam UUID[] ids, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// List entitiesIds = Arrays.stream(ids) +// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) +// .collect(Collectors.toList()); +// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); +// } +// +// +// +// @GetMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityVersions(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityTypeVersions(@PathVariable EntityType entityType, +// @RequestParam String branch) throws Exception { +// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listVersions(@RequestParam String branch) throws Exception { +// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); +// } +// +// +// +// @GetMapping("/files/version/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listFilesAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); +// } +// +// +// +// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); +// } +// +// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); +// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// return result; +// } +// +// @PostMapping("/entity/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List>> loadAllAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// SecurityUser user = getCurrentUser(); +// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); +// resultList.forEach(result -> { +// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// }); +// return resultList; +// } +// +// +// +// @GetMapping("/branches") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public Set getAllowedBranches() throws ThingsboardException { +// return versionControlService.getAllowedBranches(getTenantId()); +// } +// +// +// @PostMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { +// versionControlService.saveSettings(settings); +// } +// +// @GetMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public EntitiesVersionControlSettings getSettings() { +// return versionControlService.getSettings(); +// } +// +// +// +// @PostMapping("/repository/reset") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void resetLocalRepository() throws Exception { +// versionControlService.resetRepository(); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..a298d64ccc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -0,0 +1,395 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.utils.GitRepository; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + + private final EntitiesExportImportService exportImportService; + + private GitRepository repository; + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); + + private ScheduledExecutorService fetchExecutor; + private ScheduledFuture fetchTask; + + private final AdminSettingsService adminSettingsService; + private static final String SETTINGS_KEY = "vc"; + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + + + @AfterStartUp + public void init() { + EntitiesVersionControlSettings settings = getSettings(); + if (settings != null) { + try { + initRepository(settings); + } catch (Exception e) { + log.debug("Failed to init repository", e); + } + } + + int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); + fetchExecutor = Executors.newSingleThreadScheduledExecutor(); + fetchTask = scheduleFetch(fetchPeriod); + } + + + @Override + public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); + } + + @Override + public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + checkRepository(); + checkBranch(user.getTenantId(), branch); + + List> entityDataList = new ArrayList<>(); + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(settings.isSaveRelations()) + .build(); + for (EntityId entityId : entitiesIds) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout(branch); + repository.merge(branch); + } else { + repository.createAndCheckoutOrphanBranch(branch); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); + } + + GitRepository.Commit commit = repository.commit(versionName, "."); + repository.push(); + return toVersion(commit); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @Override + public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + } + + @Override + public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch) throws Exception { + return listVersions(tenantId, branch, null); + } + + private List listVersions(TenantId tenantId, String branch, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + } + + @Override + public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + checkVersion(tenantId, branch, versionId, path); + + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + info.setEntityName(entityData.getEntity().getName()); + return info; + }) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); + } + + @Override + public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); + } + + @Override + public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, null, settings); + } + + private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { + List> entityDataList = new ArrayList<>(); + repositoryLock.readLock().lock(); + try { + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { + EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + entityDataList.add(entityData); + } + } finally { + repositoryLock.readLock().unlock(); + } + + EntityImportSettings importSettings = EntityImportSettings.builder() + .updateRelations(settings.isLoadRelations()) + .findExistingByName(settings.isFindExistingEntityByName()) + .build(); + List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); + + return importResults.stream() + .map(importResult -> EntityVersionLoadResult.builder() + .previousEntityVersion(importResult.getOldEntity()) + .newEntityVersion(importResult.getSavedEntity()) + .entityType(importResult.getEntityType()) + .build()) + .collect(Collectors.toList()); + } + + @SneakyThrows + private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { + repositoryLock.readLock().lock(); + try { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + private void fetch() throws GitAPIException { + repositoryLock.writeLock().lock(); + try { + repository.fetch(); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private ScheduledFuture scheduleFetch(int fetchPeriod) { + return fetchExecutor.scheduleWithFixedDelay(() -> { + if (repository == null) return; + try { + fetch(); + } catch (Exception e) { + log.error("Failed to fetch remote repository", e); + } + }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + } + + + private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Version not found"); + } + } + + @Override + public List listAllowedBranches(TenantId tenantId) { + return Optional.ofNullable(getSettings()) + .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) + .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) + .orElse(Collections.emptyList()); + } + + private void checkBranch(TenantId tenantId, String branch) { + if (!listAllowedBranches(tenantId).contains(branch)) { + throw new IllegalArgumentException("Tenant does not have access to the branch"); + } + } + + + private void checkRepository() { + if (repository == null) { + throw new IllegalStateException("Repository is not initialized"); + } + } + + private void initRepository(EntitiesVersionControlSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { + this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(Path.of(settings.getRepositoryDirectory())); + this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), + settings.getUsername(), settings.getPassword()); + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private void clearRepository() throws IOException { + repositoryLock.writeLock().lock(); + try { + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repository = null; + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @SneakyThrows + @Override + public void saveSettings(EntitiesVersionControlSettings settings) { + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .orElseGet(() -> { + AdminSettings newAdminSettings = new AdminSettings(); + newAdminSettings.setKey(SETTINGS_KEY); + return newAdminSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); + + repositoryLock.writeLock().lock(); + try { + clearRepository(); + initRepository(settings); + } finally { + repositoryLock.writeLock().unlock(); + } + + if (settings.getFetchPeriod() != 0) { + fetchTask.cancel(true); + fetchTask = scheduleFetch(settings.getFetchPeriod()); + } + } + + @Override + public EntitiesVersionControlSettings getSettings() { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + .orElse(null); + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java new file mode 100644 index 0000000000..435c661b02 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; + +import java.util.List; + +public interface EntitiesVersionControlService { + + EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + + List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + + List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + + List listVersions(TenantId tenantId, String branch) throws Exception; + + + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + + List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + + List listAllowedBranches(TenantId tenantId); + + + void saveSettings(EntitiesVersionControlSettings settings); + + EntitiesVersionControlSettings getSettings(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 0d97bed2d1..06e0a9c7aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -13,20 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; +import java.util.UUID; @Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class GitSettings { - private String repositoryUri; +public class EntitiesVersionControlSettings { private String repositoryDirectory; + private String repositoryUri; private String username; private String password; + + private int fetchPeriod; + + private Map> tenantsAllowedBranches; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java index 42a2e2f555..3547f89e3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@NoArgsConstructor @AllArgsConstructor +@NoArgsConstructor public class EntityVersion { private String id; private String name; - private String authorName; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java index 6a46606845..4cf0ee2b0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java @@ -13,16 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; - -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; @Data -public class EntitiesVersionControlSettings { - private Map> allowedBranches; - private GitSettings gitSettings; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionLoadResult { + private ExportableEntity newEntityVersion; + private ExportableEntity previousEntityVersion; + private EntityType entityType; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java index 7567dba1c5..842f6ddd4e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Commit { - private final String id; - private final String message; - private final String authorName; +public class EntityVersionLoadSettings { + private boolean loadRelations; + private boolean findExistingEntityByName; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java index a458f9f5aa..1fc36463ba 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Branch { - private final String shortName; +public class EntityVersionSaveSettings { + private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 7572a003d2..97bb6ada12 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; @Data -public class Diff { - private final String type; - private final String oldPath; - private final String newPath; +public class VersionedEntityInfo { + private EntityId externalId; + private String entityName; + // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java deleted file mode 100644 index 32da1309fa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ /dev/null @@ -1,358 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; -import org.thingsboard.server.service.sync.vcs.data.GitSettings; -import org.thingsboard.server.utils.git.Repository; -import org.thingsboard.server.utils.git.data.Commit; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - // TODO [viacheslav]: start up only on one of the cores - - private final TenantService tenantService; - private final EntitiesExportImportService exportImportService; - private final AdminSettingsService adminSettingsService; - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private static final String SETTINGS_KEY = "vc"; - - private Repository repository; - private final Lock fetchLock = new ReentrantLock(); - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - @AfterStartUp - public void init() throws Exception { - try { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null && settings.getGitSettings() != null) { - this.repository = initRepository(settings.getGitSettings()); - } - } catch (Exception e) { - log.error("Failed to initialize entities version control service", e); - } - } - - - @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - private void fetch() throws Exception { - if (repository == null) return; - tryFetch(); - } - - - @Override - public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { - return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); - } - - @Override - public EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(false) - .exportOutboundRelations(false) - .build(); - List>> entityDataList = entitiesIds.stream() - .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) - .collect(Collectors.toList()); - - tryFetch(); - - repositoryLock.writeLock().lock(); - try { - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); - } - - for (EntityExportData> entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), - entityDataJson, StandardCharsets.UTF_8); - } - - Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); - repository.push(); - return toVersion(commit); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - - - @Override - public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); - } - - @Override - public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); - } - - @Override - public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, null, limit); - } - - private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { - repositoryLock.readLock().lock(); - try { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, path, limit).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - return repository.listFilesAtCommit(versionId); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return parseEntityData(entityDataJson); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); - return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } - - @Override - public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() - .map(entityDataFilePath -> { - String entityDataJson; - try { - entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return parseEntityData(entityDataJson); - }) - .collect(Collectors.toList()); - - return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } finally { - repositoryLock.readLock().unlock(); - } - } - - private void tryFetch() throws GitAPIException { - repositoryLock.readLock().lock(); - try { - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } - } finally { - repositoryLock.readLock().unlock(); - } - } - - - private String getRelativePathForEntity(EntityId entityId) { - return getRelativePathForEntityType(entityId.getEntityType()) - + "/" + entityId.getId() + ".json"; - } - - private String getRelativePathForEntityType(EntityType entityType) { - return entityType.name().toLowerCase(); - } - - - private void checkBranch(TenantId tenantId, String branch) { - // TODO [viacheslav]: all branches are available by default? - if (!getAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to this branch"); - } - } - - public Set getAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptySet()); - } - - private EntityVersion toVersion(Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); - } - - private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); - } - - - - @Override - public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .orElseGet(() -> { - AdminSettings newSettings = new AdminSettings(); - newSettings.setKey(SETTINGS_KEY); - return newSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - } - - @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) - .orElse(null); - } - - - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } - } - - private static Repository initRepository(GitSettings gitSettings) throws Exception { - if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { - return Repository.open(gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } else { - Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); - return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } - } - - public void resetRepository() throws Exception { - repositoryLock.writeLock().lock(); - try { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; - } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java deleted file mode 100644 index e05ad17826..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.List; - -public interface EntitiesVersionControlService { - - EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; - - EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception; - - - List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; - - List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; - - List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - - - List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - void saveSettings(EntitiesVersionControlSettings settings) throws Exception; - - EntitiesVersionControlSettings getSettings(); - -} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java similarity index 82% rename from application/src/main/java/org/thingsboard/server/utils/git/Repository.java rename to application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 23eb7d7db4..205bbca50e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git; +package org.thingsboard.server.utils; import com.google.common.collect.Streams; +import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.Git; @@ -32,19 +33,21 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.utils.git.data.Commit; +import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -public class Repository { +public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; @@ -52,13 +55,13 @@ public class Repository { @Getter private final String directory; - private Repository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; this.directory = directory; } - public static Repository clone(String uri, String directory, + public static GitRepository clone(String uri, String directory, String username, String password) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() @@ -67,12 +70,12 @@ public class Repository { .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new Repository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory); } - public static Repository open(String directory, String username, String password) throws IOException { + public static GitRepository open(String directory, String username, String password) throws IOException { Git git = Git.open(new java.io.File(directory)); - return new Repository(git, newCredentialsProvider(username, password), directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory); } @@ -81,6 +84,20 @@ public class Repository { .setRemoveDeletedRefs(true)); } + public void checkout(String branch) throws GitAPIException { + execute(git.checkout() + .setName(branch)); + } + + public void merge(String branch) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + public List listBranches() throws GitAPIException { return execute(git.branchList() @@ -92,12 +109,12 @@ public class Repository { } - public List listCommits(String branchName, int limit) throws IOException, GitAPIException { - return listCommits(branchName, null, limit); + public List listCommits(String branch, int limit) throws IOException, GitAPIException { + return listCommits(branch, null, limit); } - public List listCommits(String branchName, String path, int limit) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); + public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } @@ -150,20 +167,6 @@ public class Repository { } - public void checkout(String branchName) throws GitAPIException { - execute(git.checkout() - .setName(branchName)); - } - - public void merge(String branchName) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); - if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); - } - execute(git.merge() - .include(branchId)); - } - public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) @@ -177,15 +180,11 @@ public class Repository { execute(git.clean()); } - public void clean() throws GitAPIException { - execute(git.clean().setCleanDirectories(true)); - } - public Commit commit(String message, String filePattern, String author) throws GitAPIException { - execute(git.add().addFilepattern(filePattern)); + public Commit commit(String message, String filesPattern) throws GitAPIException { + execute(git.add().addFilepattern(filesPattern)); RevCommit revCommit = execute(git.commit() - .setMessage(message) - .setAuthor(author, author)); + .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); } @@ -239,12 +238,8 @@ public class Repository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand) { + if (command instanceof TransportCommand && credentialsProvider != null) { ((TransportCommand) command).setCredentialsProvider(credentialsProvider); -// SshSessionFactory sshSessionFactory = SshSessionFactory.getInstance(); -// transportCommand.setTransportConfigCallback(transport -> { -// ((SshTransport) transport).setSshSessionFactory(sshSessionFactory); -// }); } return command.call(); } @@ -253,4 +248,12 @@ public class Repository { return new UsernamePasswordCredentialsProvider(username, password); } + + @Data + public static class Commit { + private final String id; + private final String message; + private final String authorName; + } + } From 074ea8af6cdd9d723f57dd55615cf602d16738a8 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 16 May 2022 16:09:00 +0300 Subject: [PATCH 037/156] VC refactoring --- .../server/controller/AssetController.java | 4 +- .../server/controller/DeviceController.java | 4 +- .../server/controller/EdgeController.java | 4 +- .../EntitiesExportImportController.java | 206 -------- .../EntitiesVersionControlController.java | 293 ++++++----- .../service/asset/AssetBulkImportService.java | 4 +- .../device/DeviceBulkImportService.java | 4 +- .../service/edge/EdgeBulkImportService.java | 4 +- .../DefaultEntitiesExportImportService.java | 30 +- .../EntitiesExportImportService.java | 10 +- .../DefaultExportableEntitiesService.java | 56 ++- .../exporting/EntityExportService.java | 6 +- .../exporting/ExportableEntitiesService.java | 12 +- .../exporting/data/DeviceExportData.java | 2 +- .../exporting/data/EntityExportData.java | 2 +- .../exporting/data}/EntityExportSettings.java | 2 +- .../exporting/data/RuleChainExportData.java | 2 +- .../impl/BaseEntityExportService.java | 6 +- .../impl/DefaultEntityExportService.java | 10 +- .../exporting/impl/DeviceExportService.java | 4 +- .../impl/RuleChainExportService.java | 4 +- .../importing/EntityImportService.java | 8 +- .../csv/AbstractBulkImportService.java | 9 +- .../importing/csv/BulkImportColumnType.java | 2 +- .../importing/csv/BulkImportRequest.java | 2 +- .../importing/csv/BulkImportResult.java | 2 +- .../importing/csv/ImportedEntityInfo.java | 2 +- .../importing/data/EntityImportResult.java | 2 +- .../importing/data/EntityImportSettings.java | 2 +- .../importing/impl/AssetImportService.java | 4 +- .../impl/BaseEntityImportService.java | 12 +- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 4 +- .../impl/DeviceProfileImportService.java | 4 +- .../impl/RuleChainImportService.java | 6 +- .../importing/data/request/ImportRequest.java | 32 -- .../DefaultEntitiesVersionControlService.java | 463 +++++++++++------- .../vc/EntitiesVersionControlService.java | 27 +- .../data/EntitiesVersionControlSettings.java | 10 +- .../sync/vc/data/VersionCreationResult.java | 11 + ...LoadResult.java => VersionLoadResult.java} | 8 +- ...iesByCustomFilterVersionCreateConfig.java} | 14 +- ...tiesByCustomQueryVersionCreateConfig.java} | 12 +- .../EntityListVersionCreateConfig.java} | 10 +- .../EntityTypeVersionCreateConfig.java} | 14 +- .../SingleEntityVersionCreateConfig.java} | 10 +- .../request/create/VersionCreateConfig.java} | 18 +- .../create/VersionCreateConfigType.java} | 4 +- .../create/VersionCreateRequest.java} | 14 +- .../load/EntityTypeVersionLoadConfig.java | 12 + .../load/EntityTypeVersionLoadRequest.java | 20 + .../request/load/EntityVersionLoadConfig.java | 11 + .../load/SingleEntityVersionLoadRequest.java | 20 + .../load/VersionLoadRequest.java} | 11 +- .../request/load/VersionLoadRequestType.java | 6 + .../server/utils/GitRepository.java | 39 +- ...aseEntitiesExportImportControllerTest.java | 20 +- ...EntitiesExportImportControllerSqlTest.java | 53 +- .../org/thingsboard/server/dao/DaoUtil.java | 20 +- .../server/dao/ExportableEntityDao.java | 4 + .../server/dao/sql/asset/JpaAssetDao.java | 5 + .../dao/sql/customer/JpaCustomerDao.java | 5 + .../sql/dashboard/DashboardRepository.java | 4 + .../dao/sql/dashboard/JpaDashboardDao.java | 7 + .../server/dao/sql/device/JpaDeviceDao.java | 5 + .../dao/sql/device/JpaDeviceProfileDao.java | 5 + .../server/dao/sql/rule/JpaRuleChainDao.java | 5 + 68 files changed, 860 insertions(+), 777 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/DefaultEntitiesExportImportService.java (84%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/EntitiesExportImportService.java (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/DefaultExportableEntitiesService.java (77%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/EntityExportService.java (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/ExportableEntitiesService.java (74%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/DeviceExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/EntityExportData.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request => exportimport/exporting/data}/EntityExportSettings.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/data/RuleChainExportData.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/BaseEntityExportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/DefaultEntityExportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/EntityImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/AbstractBulkImportService.java (97%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportColumnType.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportRequest.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/BulkImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/csv/ImportedEntityInfo.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/data/EntityImportResult.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/data/EntityImportSettings.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/AssetImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/BaseEntityImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/CustomerImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DeviceImportService.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/DeviceProfileImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{ => exportimport}/importing/impl/RuleChainImportService.java (95%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionLoadResult.java => VersionLoadResult.java} (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/CustomEntityFilterExportRequest.java => vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java} (69%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/CustomEntityQueryExportRequest.java => vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java} (72%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/EntityListExportRequest.java => vc/data/request/create/EntityListVersionCreateConfig.java} (73%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/EntityTypeExportRequest.java => vc/data/request/create/EntityTypeVersionCreateConfig.java} (74%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/SingleEntityExportRequest.java => vc/data/request/create/SingleEntityVersionCreateConfig.java} (77%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/ExportRequest.java => vc/data/request/create/VersionCreateConfig.java} (58%) rename application/src/main/java/org/thingsboard/server/service/sync/{exporting/data/request/ExportRequestType.java => vc/data/request/create/VersionCreateConfigType.java} (87%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionLoadSettings.java => request/create/VersionCreateRequest.java} (73%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/data/{EntityVersionSaveSettings.java => request/load/VersionLoadRequest.java} (74%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index e48f2dcf23..66b570cf1d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -54,8 +54,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 03f0996ea9..4b6eca7879 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -74,8 +74,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8f45427603..948a380fcd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -55,8 +55,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java deleted file mode 100644 index c91fd0c292..0000000000 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.controller; - -import io.swagger.annotations.ApiOperation; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; - -@RestController -@RequestMapping("/api/entities") -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class EntitiesExportImportController extends BaseController { - - private final EntitiesExportImportService exportImportService; - private final ExportableEntitiesService exportableEntitiesService; - - - @ApiOperation(value = "Export entities by request", notes = "" + - "Takes export request and returns list of export data for each entity found by request. " + - "Supported entity types for export, hence for import, are **DEVICE**, **DEVICE_PROFILE**, **ASSET**, " + - "**CUSTOMER**, **RULE_CHAIN** and **DASHBOARD**." + NEW_LINE + - "For each type of export request, you can set some export settings: \n" + - "- **exportRelations** - whether to export inbound and outbound relations for an entity " + - "(only relations of type group COMMON can be exported)" + NEW_LINE + - "Supported export requests:\n" + - "- **SINGLE_ENTITY**:" + NEW_LINE + - " To export a single entity by id. Example:" + NEW_LINE + - "```\n{\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "- **ENTITY_LIST**:" + NEW_LINE + - " To export a list of entities by their ids. Example:" + NEW_LINE + - "```\n{\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"DEVICE\",\n \"id\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"ASSET\",\n \"id\": \"2f0a3bd0-989d-11ec-93b5-6de6c2b68078\"\n }\n ],\n \"exportSettings\": {\n \"exportRelations\": true\n }\n}\n```" + NEW_LINE + - "- **ENTITY_TYPE**:" + NEW_LINE + - " To export entities of specified entity type. You need to specify page size, " + - "and may specify page index and customer id (to limit the list of entities to the ones owned by a customer). " + - "Entities are ordered by created time descendingly. Example:" + NEW_LINE + - "```\n{\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": \"2eb16d70-989d-11ec-93b5-6de6c2b68078\"\n}\n```" + NEW_LINE + - "- **CUSTOM_ENTITY_FILTER**:" + NEW_LINE + - " To export entities by custom entity filter. The order used is the same as for ENTITY_TYPE export request. Example:" + NEW_LINE + - "```\n{\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"Thermostats\",\n \"deviceNameFilter\": \"\"\n },\n \"page\": 0,\n \"pageSize\": 100,\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "- **CUSTOM_ENTITY_QUERY**:" + NEW_LINE + - " To export entities by custom entity query. Example: " + NEW_LINE + - "```\n{\n \"type\": \"CUSTOM_ENTITY_QUERY\",\n \"query\": {\n \"entityFilter\": {\n \"type\": \"entityType\",\n \"entityType\": \"DEVICE\"\n },\n \"pageLink\": {\n \"page\": 0,\n \"pageSize\": 200,\n \"textSearch\": \"THB_\",\n \"sortOrder\": {\n \"key\": {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n },\n \"direction\": \"DESC\"\n }\n },\n \"entityFields\": [\n {\n \"type\": \"ENTITY_FIELD\",\n \"key\": \"name\"\n }\n ],\n \"latestValues\": [\n {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n }\n ],\n \"keyFilters\": [\n {\n \"key\": {\n \"type\": \"SERVER_ATTRIBUTE\",\n \"key\": \"lastActivityTime\"\n },\n \"valueType\": \"NUMERIC\",\n \"predicate\": {\n \"type\": \"NUMERIC\",\n \"operation\": \"GREATER\",\n \"value\": {\n \"defaultValue\": 0\n }\n }\n }\n ]\n },\n \"customerId\": null,\n \"exportSettings\": {\n \"exportRelations\": false\n }\n}\n```" + NEW_LINE + - "Mostly, export data of an entity contains the whole entity itself and its relations " + - "(if option to export relations was enabled):" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"ASSET\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1648204424029,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": { ... },\n \"customerId\": { ... },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n ...\n },\n \"relations\": [\n {\n \"from\": {\n \"entityType\": \"ASSET\",\n \"id\": ...\n },\n \"to\": {\n \"entityType\": \"DEVICE\",\n \"id\": ...\n },\n \"type\": \"Contains\",\n \"typeGroup\": \"COMMON\",\n \"additionalInfo\": {\n \"a\": \"b\"\n }\n }\n ]\n }\n]\n```" + NEW_LINE + - "For devices, export data will additionally contain device's credentials; for rule chains - its metadata:" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"DEVICE\",\n \"entity\": { ... },\n \"credentials\": {\n \"id\": { ... },\n \"createdTime\": 1648829321209,\n \"deviceId\": { ... },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"5cZEDo45KGW7JgVNv4Ko\",\n \"credentialsValue\": null\n }\n }\n]\n```" + NEW_LINE + - "```\n[\n {\n \"entityType\": \"RULE_CHAIN\",\n \"entity\": {\n \"id\": { ... },\n \"createdTime\": 1646056614257,\n \"additionalInfo\": null,\n \"tenantId\": { ... },\n \"name\": \"Rule Chain 2\",\n \"type\": \"CORE\",\n \"firstRuleNodeId\": { ... },\n \"root\": false,\n ...\n },\n \"metaData\": {\n \"ruleChainId\": { ... },\n \"firstNodeIndex\": 7,\n \"nodes\": [ ... ],\n \"connections\": [ ... ],\n \"ruleChainConnections\": null\n }\n }\n]\n```" + NEW_LINE + - "Returned export data is to be used later for import request." + NEW_LINE + - "If any entity found by request is of unsupported type - an error will be returned.\n" + - "Also, if a user does not have a READ permission for an entity (or, if relations are exported, for a bounded entity), " + - "access will be denied." + - ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) - @PostMapping("/export") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntities(@RequestBody ExportRequest exportRequest) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - return exportEntitiesByRequest(user, exportRequest); - } catch (Exception e) { - log.warn("Failed to export entities for request {}", exportRequest, e); - throw handleException(e); - } - } - - @ApiOperation(value = "Export entities by multiple requests", notes = "" + - "The API behaviour is the same as for exporting entities by single request, " + - "except that this method takes an array of export requests as a request body." + NEW_LINE + - "Example:" + NEW_LINE + - "```\n[\n {\n \"type\": \"SINGLE_ENTITY\",\n \"entityId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"5f9eda10-b442-11ec-bbf5-adec34031568\"\n }\n },\n {\n \"type\": \"CUSTOM_ENTITY_FILTER\",\n \"filter\": {\n \"type\": \"deviceType\",\n \"deviceType\": \"thermostat\",\n \"deviceNameFilter\": \"\"\n },\n \"pageSize\": 1000\n },\n {\n \"type\": \"ENTITY_TYPE\",\n \"entityType\": \"ASSET\",\n \"pageSize\": 1000,\n \"exportSettings\": {\n \"exportRelations\": true\n }\n },\n {\n \"type\": \"ENTITY_LIST\",\n \"entitiesIds\": [\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"2ef13590-989d-11ec-93b5-6de6c2b68078\"\n },\n {\n \"entityType\": \"RULE_CHAIN\",\n \"id\": \"e7311ec0-b442-11ec-bbf5-adec34031568\"\n }\n ]\n }\n]\n```" + - ControllerConstants.TENANT_AUTHORITY_PARAGRAPH) - @PostMapping(value = "/export", params = {"multiple"}) - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List> exportEntities(@RequestBody List exportRequests) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - List> result = new ArrayList<>(); - for (ExportRequest exportRequest : exportRequests) { - List> exportDataList = exportEntitiesByRequest(user, exportRequest); - result.addAll(exportDataList); - } - return result; - } catch (Exception e) { - log.warn("Failed to export entities for requests {}", exportRequests, e); - throw handleException(e); - } - } - - private List> exportEntitiesByRequest(SecurityUser user, ExportRequest exportRequest) throws ThingsboardException { - List entities = exportableEntitiesService.findEntitiesForRequest(user.getTenantId(), exportRequest); - - EntityExportSettings exportSettings = exportRequest.getExportSettings(); - if (exportSettings == null) { - exportSettings = EntityExportSettings.builder() - .exportRelations(false) - .build(); - } - - List> exportDataList = new ArrayList<>(); - for (EntityId entityId : entities) { - EntityExportData exportData = exportImportService.exportEntity(user, entityId, exportSettings); - exportDataList.add(exportData); - } - return exportDataList; - } - - - @ApiOperation(value = "Import entities by request", notes = "" + - "Takes import request and returns the list of import results. " + - "Import request must contain the list of export data and might contain import settings. " + NEW_LINE + - "The method creates an entity if it is new in the scope of a tenant, or otherwise updates an existing one. " + - "On entity import request, we first try to find an entity within a tenant that has externalId equal " + - "to the id in the export data. If the platform fails to do that, we then search for an entity with " + - "regular (internal) id like the one in the export data (this is useful in case we are exporting and " + - "importing entities within the same tenant). Then, if we still haven't found any entity, if findExistingByName " + - "option of the EntityImportSettings is enabled, we will search for the one by its name (this is also useful " + - "for avoiding conflicts with default device profile or Root Rule Chain when importing all entities from another " + - "tenant). After, if the exported entity is new for this tenant, we simply save it with external id " + - "from the export data, and also create relations if any (if updateRelations option is enabled). " + - "Otherwise, we will reset all fields of the existing entity to the ones from the export data and save it, " + - "and also will update the list of relations (remove the ones that aren't present in the export data, " + - "and update or create others), if updateRelations option is enabled." + NEW_LINE + - "If an entity contains references to some other entities, like device references certain device profile, " + - "we will find this other entity within the tenant by this principle: look for an entity with " + - "such external id, or otherwise, internal id. This requires referenced entities to be imported " + - "before the referencing entity (if we are importing to another tenant). So, when receiving the list " + - "of entities' export data for import, we first try to fix the data order to import 'standalone' entities first." + NEW_LINE + - "As for relations importing, they are processed after all entities in the import batch are already saved, " + - "and the internal id of a bounded entity is found with the regular principle." + NEW_LINE + - "Import of all entities and their relations from the import request is processed in the single transaction, " + - "and so everything will be rolled back if the platform fails to e.g. find internal entity by external id. \n" + - "Example of import request:\n" + - "```\n{\n \"importSettings\": {\n \"findExistingByName\": false,\n \"updateRelations\": false\n },\n \"exportDataList\": [\n {\n \"entityType\": \"DEVICE_PROFILE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n \"createdTime\": 1649096026765,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"name\": \"Profile 1\",\n ...\n }\n },\n {\n \"entityType\": \"DEVICE\",\n \"entity\": {\n \"id\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154276962,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"4c9001b0-b442-11ec-bbf5-adec34031568\"\n },\n \"customerId\": {\n \"entityType\": \"CUSTOMER\",\n \"id\": \"13814000-1dd2-11b2-8080-808080808080\"\n },\n \"name\": \"Device 1\",\n \"type\": \"Profile 1\",\n \"label\": \"v1.0\",\n \"deviceProfileId\": {\n \"entityType\": \"DEVICE_PROFILE\",\n \"id\": \"f84363d0-b442-11ec-bbf5-adec34031568\"\n },\n ...\n },\n \"credentials\": {\n \"id\": {\n \"id\": \"981e0360-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"createdTime\": 1649154277014,\n \"deviceId\": {\n \"entityType\": \"DEVICE\",\n \"id\": \"98161420-b4ca-11ec-ab0c-e7744c90d468\"\n },\n \"credentialsType\": \"ACCESS_TOKEN\",\n \"credentialsId\": \"sGExNdnl71uKmkNvtNdp\",\n \"credentialsValue\": null\n }\n }\n ]\n}\n```" + NEW_LINE + - "The response contains a list of EntityImportResult which has values of savedEntity and oldEntity:\n" + - "```\n[\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"additionalInfo\": {\n \"description\": \"\"\n },\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v2.0\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"6b03ab20-989e-11ec-b446-89df822d7fa2\"\n }\n },\n \"oldEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"d73d7690-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"createdTime\": 1649166408825,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 1\",\n \"type\": \"A\",\n \"label\": \"v1.0\",\n ...\n \"externalId\": null\n },\n \"entityType\": \"ASSET\"\n },\n {\n \"savedEntity\": {\n \"id\": {\n \"entityType\": \"ASSET\",\n \"id\": \"387213f0-b4ea-11ec-abfc-6dcc6508d0b5\"\n },\n \"createdTime\": 1649167860399,\n \"tenantId\": {\n \"entityType\": \"TENANT\",\n \"id\": \"c0b2e4f0-b4e6-11ec-b9eb-0562e1a20a1b\"\n },\n \"name\": \"Asset 2\",\n \"type\": \"B\",\n ...\n \"externalId\": {\n \"entityType\": \"ASSET\",\n \"id\": \"0b9ea0d0-ac27-11ec-a2a6-89d15eae3b21\"\n }\n },\n \"oldEntity\": null,\n \"entityType\": \"ASSET\"\n }\n]\n```") - @PostMapping("/import") - public List> importEntities(@RequestBody ImportRequest importRequest) throws ThingsboardException { - SecurityUser user = getCurrentUser(); - try { - EntityImportSettings importSettings = importRequest.getImportSettings(); - if (importSettings == null) { - importSettings = EntityImportSettings.builder() - .findExistingByName(false) - .updateRelations(false) - .build(); - } - - List> importResults = exportImportService.importEntities(user, importRequest.getExportDataList(), importSettings); - - importResults.stream() - .map(EntityImportResult::getSendEventsCallback) - .filter(Objects::nonNull) - .forEach(sendEventsCallback -> { - try { - sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send event for entity", e); - } - }); - - return importResults; - } catch (Exception e) { - log.warn("Failed to import entities for request {}", importRequest, e); - throw handleException(e); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index c39442341e..0625dbeb38 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,140 +15,189 @@ */ package org.thingsboard.server.controller; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; @RestController @RequestMapping("/api/entities/vc") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; - // search request - export request with settings - -// -// @PostMapping("/version/{entityType}/{entityId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID id, -// @RequestParam String branch, -// @RequestBody String versionName) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); -// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); -// } -// -// @PostMapping("/version/{entityType}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, -// @RequestParam UUID[] ids, -// @RequestParam String branch, -// @RequestBody String versionName) throws Exception { -// List entitiesIds = Arrays.stream(ids) -// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) -// .collect(Collectors.toList()); -// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); -// } -// -// -// -// @GetMapping("/version/{entityType}/{entityId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listEntityVersions(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); -// } -// -// @GetMapping("/version/{entityType}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listEntityTypeVersions(@PathVariable EntityType entityType, -// @RequestParam String branch) throws Exception { -// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); -// } -// -// @GetMapping("/version") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listVersions(@RequestParam String branch) throws Exception { -// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); -// } -// -// -// -// @GetMapping("/files/version/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List listFilesAtVersion(@RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); -// } -// -// -// -// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); -// } -// -// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, -// @PathVariable("entityId") UUID entityUuid, -// @RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); -// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); -// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); -// return result; -// } -// -// @PostMapping("/entity/{versionId}") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public List>> loadAllAtVersion(@RequestParam String branch, -// @PathVariable String versionId) throws Exception { -// SecurityUser user = getCurrentUser(); -// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); -// resultList.forEach(result -> { -// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); -// }); -// return resultList; -// } -// -// -// -// @GetMapping("/branches") -// @PreAuthorize("hasAuthority('TENANT_ADMIN')") -// public Set getAllowedBranches() throws ThingsboardException { -// return versionControlService.getAllowedBranches(getTenantId()); -// } -// -// -// @PostMapping("/settings") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { -// versionControlService.saveSettings(settings); -// } -// -// @GetMapping("/settings") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public EntitiesVersionControlSettings getSettings() { -// return versionControlService.getSettings(); -// } -// -// -// -// @PostMapping("/repository/reset") -// @PreAuthorize("hasAuthority('SYS_ADMIN')") -// public void resetLocalRepository() throws Exception { -// versionControlService.resetRepository(); -// } + @PostMapping("/version") + public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + return versionControlService.saveEntitiesVersion(user, request); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") + public List listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { + try { + EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/version/{branch}/{entityType}") + public List listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType) throws ThingsboardException { + try { + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/version/{branch}") + public List listVersions(@PathVariable String branch) throws ThingsboardException { + try { + return versionControlService.listVersions(getTenantId(), branch); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/entity/{branch}/{entityType}/{versionId}") + public List listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { + try { + return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/entity/{branch}/{versionId}") + public List listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { + try { + return versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId); + } catch (Exception e) { + throw handleException(e); + } + } + + + @PostMapping("/entity") + public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + String versionId = request.getVersionId(); + if (versionId == null) { + List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); + if (versions.size() > 0) { + versionId = versions.get(0).getId(); + } else { + throw new IllegalArgumentException("No versions available in branch"); + } + } + + return versionControlService.loadEntitiesVersion(user, request); + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiModelProperty(notes = "" + + "") + @GetMapping("/branches") + public List listBranches() throws ThingsboardException { + try { + List remoteBranches = versionControlService.listBranches(getTenantId()); + List infos = new ArrayList<>(); + + String defaultBranch = getSettings().getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + remoteBranches.remove(defaultBranch); + infos.add(new BranchInfo(defaultBranch, true)); + } + + remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + return infos; + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiModelProperty(notes = "" + + "```\n{\n" + + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + + " \"username\": \"User\",\n" + + " \"password\": \"api_key\",\n" + + " \"defaultBranch\": \"master\"\n" + + "}\n```") + @GetMapping("/settings") + public EntitiesVersionControlSettings getSettings() throws ThingsboardException { + try { + return versionControlService.getSettings(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiModelProperty(notes = "" + + "```\n{\n" + + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + + " \"username\": \"User\",\n" + + " \"password\": \"api_key\",\n" + + " \"defaultBranch\": \"master\"\n" + + "}\n```") + @PostMapping("/settings") + public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + versionControlService.saveSettings(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + + + @Data + public static class BranchInfo { + private final String name; + private final boolean isDefault; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index cfefcafbf4..179c9c697d 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index 3aac9c3641..3e63652b97 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index b09ac532df..75f7133721 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -25,8 +25,8 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java index 55384b21db..608a256514 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync; +package org.thingsboard.server.service.sync.exportimport; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,14 +27,14 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import org.thingsboard.server.utils.ThrowingRunnable; import java.util.ArrayList; @@ -74,6 +74,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { fixOrder(exportDataList); + List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { @@ -88,6 +89,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS saveReferencesCallback.run(); } + importResults.stream() + .map(EntityImportResult::getSendEventsCallback) + .filter(Objects::nonNull) + .forEach(sendEventsCallback -> { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send event for entity", e); + } + }); + return importResults; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java index eaaf01ba2c..5c29feb57c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync; +package org.thingsboard.server.service.sync.exportimport; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java index ce6924de93..4d772ad935 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,6 +30,8 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -37,7 +39,6 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityFilter; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.entity.EntityService; @@ -46,12 +47,12 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityQueryExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; import java.util.Collection; import java.util.Collections; @@ -129,6 +130,16 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi return entity; } + @Override + public , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { + Dao dao = getDao(entityType); + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + return exportableEntityDao.findByTenantId(tenantId.getId(), pageLink); + } + return new PageData<>(); + } + private boolean belongsToTenant(HasId entity, TenantId tenantId) { return tenantId.equals(((HasTenantId) entity).getTenantId()); } @@ -136,31 +147,31 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi @Transactional(readOnly = true, timeout = 40) @Override - public List findEntitiesForRequest(TenantId tenantId, ExportRequest request) { + public List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig request) { switch (request.getType()) { case SINGLE_ENTITY: { - return List.of(((SingleEntityExportRequest) request).getEntityId()); + return List.of(((SingleEntityVersionCreateConfig) request).getEntityId()); } case ENTITY_LIST: { - return ((EntityListExportRequest) request).getEntitiesIds(); + return ((EntityListVersionCreateConfig) request).getEntitiesIds(); } case ENTITY_TYPE: { - EntityTypeExportRequest exportRequest = (EntityTypeExportRequest) request; - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + EntityTypeVersionCreateConfig exportRequest = (EntityTypeVersionCreateConfig) request; + org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); entityTypeFilter.setEntityType(exportRequest.getEntityType()); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, exportRequest.getPage(), exportRequest.getPageSize()); + return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, 0, Integer.MAX_VALUE); } case CUSTOM_ENTITY_FILTER: { - CustomEntityFilterExportRequest exportRequest = (CustomEntityFilterExportRequest) request; + EntitiesByCustomFilterVersionCreateConfig exportRequest = (EntitiesByCustomFilterVersionCreateConfig) request; EntityFilter filter = exportRequest.getFilter(); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, filter, exportRequest.getPage(), exportRequest.getPageSize()); + return findEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE); } case CUSTOM_ENTITY_QUERY: { - CustomEntityQueryExportRequest exportRequest = (CustomEntityQueryExportRequest) request; + EntitiesByCustomQueryVersionCreateConfig exportRequest = (EntitiesByCustomQueryVersionCreateConfig) request; EntityDataQuery query = exportRequest.getQuery(); CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); @@ -194,6 +205,17 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } } + @Override + public void deleteByTenantIdAndId(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + + dao.removeById(tenantId, id.getId()); + } + @Override public void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java index 73f946e722..5532f30125 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java index 9021a03e85..043ec91e20 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting; +package org.thingsboard.server.service.sync.exportimport.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,9 +21,11 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; import java.util.List; @@ -35,7 +37,11 @@ public interface ExportableEntitiesService { , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); - List findEntitiesForRequest(TenantId tenantId, ExportRequest request); + , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); + + List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig filter); // FIXME [viacheslav]: + + void deleteByTenantIdAndId(TenantId tenantId, I id); void checkPermission(SecurityUser user, HasId entity, EntityType entityType, Operation operation) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java index 56633f9c91..4be338d7cc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/DeviceExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java index 728d8212c4..81e3b344d0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java index 5454105ee5..0fb318b5fb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityExportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java index 2e05e2f1f5..e1d870473f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/RuleChainExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data; +package org.thingsboard.server.service.sync.exportimport.exporting.data; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java index 7b5f758f93..46428a763f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java index a051f0e59f..69af613480 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java index 21df8beda2..93f9c51655 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java index e814cd6ce7..f40ee6884a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.impl; +package org.thingsboard.server.service.sync.exportimport.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java index 6c82530273..be6f62efe4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing; +package org.thingsboard.server.service.sync.exportimport.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java index 06cd7a93be..b2bfafc695 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,6 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.sync.importing.csv.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -166,7 +165,7 @@ public abstract class AbstractBulkImportService data) { + private void saveKvs(SecurityUser user, E entity, Map data) { Arrays.stream(BulkImportColumnType.values()) .filter(BulkImportColumnType::isKv) .map(kvType -> { @@ -250,7 +249,7 @@ public abstract class AbstractBulkImportService columnsMappings = request.getMapping().getColumns(); + List columnsMappings = request.getMapping().getColumns(); return records.stream() .map(record -> { EntityData entityData = new EntityData(); @@ -281,7 +280,7 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); - private final Map kvs = new LinkedHashMap<>(); + private final Map kvs = new LinkedHashMap<>(); private int lineNumber; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java index beafe5e95f..9c0e6b6491 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java index f600289ab6..6347910cab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java index da12a6baeb..550d4a72f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java index 68e7a49b18..a39dcc7481 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.csv; +package org.thingsboard.server.service.sync.exportimport.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java index 62c574d5f0..242856bfcb 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.data; +package org.thingsboard.server.service.sync.exportimport.importing.data; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java index a5de7d8744..075a6896ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/EntityImportSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.data; +package org.thingsboard.server.service.sync.exportimport.importing.data; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java index ae54a988f3..80a942c73f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java index 28095a7b85..7ecfa7d3ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportService; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java index 1a79fe666b..b69c5a0d5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java index 77f32bf94c..b85e403f87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java index 091b6eb856..112152365b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java index 5bc45dc567..f97950aaf8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java index 09a13d5f14..993f9fa27a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.importing.impl; +package org.thingsboard.server.service.sync.exportimport.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java deleted file mode 100644 index 7943e1a404..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/importing/data/request/ImportRequest.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.importing.data.request; - -import lombok.Data; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; - -import java.util.List; - -@Data -public class ImportRequest { - - private List> exportDataList; - private EntityImportSettings importSettings; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index a298d64ccc..919960ce62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -18,35 +18,62 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; import org.thingsboard.server.utils.GitRepository; import java.io.File; @@ -56,16 +83,19 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -73,75 +103,152 @@ import java.util.stream.Collectors; public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { private final EntitiesExportImportService exportImportService; + private final ExportableEntitiesService exportableEntitiesService; + private final AttributesService attributesService; + private final EntityService entityService; + private final TenantDao tenantDao; - private GitRepository repository; - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - private ScheduledExecutorService fetchExecutor; - private ScheduledFuture fetchTask; + // TODO [viacheslav]: concurrency + private final Map repositories = new ConcurrentHashMap<>(); + @Value("${java.io.tmpdir}") + private String repositoriesFolder; - private final AdminSettingsService adminSettingsService; private static final String SETTINGS_KEY = "vc"; private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); @AfterStartUp public void init() { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null) { - try { - initRepository(settings); - } catch (Exception e) { - log.debug("Failed to init repository", e); + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } } - } - - int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); - fetchExecutor = Executors.newSingleThreadScheduledExecutor(); - fetchTask = scheduleFetch(fetchPeriod); + }); + Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, 5, 5, TimeUnit.SECONDS); } @Override - public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { - return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); - } + public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + GitRepository repository = checkRepository(user.getTenantId()); + repository.getLock().writeLock().lock(); - @Override - public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { - repositoryLock.writeLock().lock(); try { - checkRepository(); - checkBranch(user.getTenantId(), branch); - - List> entityDataList = new ArrayList<>(); - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportRelations(settings.isSaveRelations()) - .build(); - for (EntityId entityId : entitiesIds) { - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); - entityDataList.add(entityData); + repository.fetch(); + if (repository.listBranches().contains(request.getBranch())) { + repository.checkout(request.getBranch()); + repository.merge(request.getBranch()); + } else { // FIXME [viacheslav]: rollback orphan branch on failure + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch } - fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); + for (VersionCreateRequest.Config config : request.getConfigs()) { + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .build(); + + List> entityDataList = new ArrayList<>(); + for (EntityId entityId : findEntities()) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + if (config.isRemoveOtherRemoteEntitiesOfType()) { + entityDataList.stream() + .map(EntityExportData::getEntityType) + .distinct() + .forEach(entityType -> { + try { + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + } } + // TODO [viacheslav]: find with pagination - for (EntityExportData entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); - } + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); - GitRepository.Commit commit = repository.commit(versionName, "."); + GitRepository.Commit commit = repository.commit(request.getVersionName()); repository.push(); - return toVersion(commit); + + result.setVersion(toVersion(commit)); + return result; } finally { - repositoryLock.writeLock().unlock(); + repository.getLock().writeLock().unlock(); + } + } + + private List findEntities(SecurityUser user, VersionCreateConfig entityFilter, int page, int pageSize) { + switch (entityFilter.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) entityFilter; + return List.of(filter.getEntityId()); + } + case ENTITY_LIST: { + EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) entityFilter; + return filter.getEntitiesIds(); + } + case ENTITY_TYPE: { + EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) entityFilter; + EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); + + org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); + entityTypeFilter.setEntityType(filter.getEntityType()); + + newFilter.setFilter(entityTypeFilter); + newFilter.setCustomerId(filter.getCustomerId()); + return findEntities(user, newFilter, page, pageSize); + } + case CUSTOM_ENTITY_FILTER: { + EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) entityFilter; + EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); + + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(filter.getFilter(), pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + newFilter.setQuery(query); + newFilter.setCustomerId(filter.getCustomerId()); + return findEntities(user, newFilter, page, pageSize); + } + case CUSTOM_ENTITY_QUERY: { + EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) entityFilter; + CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); + return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() + .stream().map(EntityData::getEntityId) + .collect(Collectors.toList()); + } } } @@ -162,16 +269,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - repositoryLock.readLock().lock(); + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { - checkRepository(); - checkBranch(tenantId, branch); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() .map(this::toVersion) .collect(Collectors.toList()); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } @@ -187,16 +292,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - repositoryLock.readLock().lock(); + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { - checkRepository(); - checkBranch(tenantId, branch); - checkVersion(tenantId, branch, versionId, path); - + checkVersion(tenantId, branch, versionId); return repository.listFilesAtCommit(versionId, path).stream() .map(filePath -> { EntityId entityId = fromRelativePath(filePath); - EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + EntityExportData entityData = getEntityDataAtVersion(tenantId, entityId, versionId); VersionedEntityInfo info = new VersionedEntityInfo(); info.setExternalId(entityId); @@ -205,171 +308,155 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }) .collect(Collectors.toList()); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } @Override - public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); - } - - @Override - public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); - } - - @Override - public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { - return loadAtVersion(user, branch, versionId, null, settings); - } + public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + GitRepository repository = checkRepository(user.getTenantId()); - private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { List> entityDataList = new ArrayList<>(); - repositoryLock.readLock().lock(); + EntityVersion version; + repository.getLock().readLock().lock(); try { - for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { - EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), path)) { + EntityExportData entityData = getEntityDataAtVersion(user.getTenantId(), info.getExternalId(), versionId); entityDataList.add(entityData); } } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } EntityImportSettings importSettings = EntityImportSettings.builder() .updateRelations(settings.isLoadRelations()) .findExistingByName(settings.isFindExistingEntityByName()) .build(); + // FIXME [viacheslav]: do evrth in transaction List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); - return importResults.stream() - .map(importResult -> EntityVersionLoadResult.builder() - .previousEntityVersion(importResult.getOldEntity()) - .newEntityVersion(importResult.getSavedEntity()) - .entityType(importResult.getEntityType()) - .build()) - .collect(Collectors.toList()); + Map results = new HashMap<>(); + + boolean removeNonexistentLocalEntities = false; + if () + if (request.isRemoveOtherLocalEntitiesOfType()) { + importResults.stream() + .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? + .forEach((entityType, resultsForEntityType) -> { + Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); + AtomicInteger deleted = new AtomicInteger(); + + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + // need to delete in a specific order? + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + deleted.getAndIncrement(); + } + }); + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) + .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) + .deleted(deleted.get()) + .build()); + }); + } + + return new ArrayList<>(results.values()); } @SneakyThrows - private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { - repositoryLock.readLock().lock(); + private EntityExportData getEntityDataAtVersion(TenantId tenantId, EntityId externalId, String versionId) { + GitRepository repository = checkRepository(tenantId); + repository.getLock().readLock().lock(); try { String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); return JacksonUtil.fromString(entityDataJson, EntityExportData.class); } finally { - repositoryLock.readLock().unlock(); + repository.getLock().readLock().unlock(); } } - - private void fetch() throws GitAPIException { - repositoryLock.writeLock().lock(); - try { - repository.fetch(); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - private ScheduledFuture scheduleFetch(int fetchPeriod) { - return fetchExecutor.scheduleWithFixedDelay(() -> { - if (repository == null) return; - try { - fetch(); - } catch (Exception e) { - log.error("Failed to fetch remote repository", e); - } - }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + private void updateEntityVersionInfo(TenantId tenantId, EntityId entityId, EntityVersion version) { + ObjectNode versionInfo = JacksonUtil.newObjectNode(); + versionInfo.set("versionName", new TextNode(version.getName())); + versionInfo.set("versionId", new TextNode(version.getId())); + attributesService.save(tenantId, entityId, DataConstants.SERVER_SCOPE, + List.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry("entityVersionInfo", versionInfo.toString())))); } - private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Version not found"); - } - } - @Override - public List listAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptyList()); + public List listBranches(TenantId tenantId) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listBranches(); } - private void checkBranch(TenantId tenantId, String branch) { - if (!listAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to the branch"); - } - } + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); } - private void initRepository(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { - this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); - } else { - Files.createDirectories(Path.of(settings.getRepositoryDirectory())); - this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), - settings.getUsername(), settings.getPassword()); - } - } finally { - repositoryLock.writeLock().unlock(); + private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + repository = GitRepository.open(repositoryDirectory.toFile(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); } + repositories.put(tenantId, repository); } - private void clearRepository() throws IOException { - repositoryLock.writeLock().lock(); - try { - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repository = null; - } - } finally { - repositoryLock.writeLock().unlock(); + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); } } @SneakyThrows @Override - public void saveSettings(EntitiesVersionControlSettings settings) { - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) - .orElseGet(() -> { - AdminSettings newAdminSettings = new AdminSettings(); - newAdminSettings.setKey(SETTINGS_KEY); - return newAdminSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - - repositoryLock.writeLock().lock(); - try { - clearRepository(); - initRepository(settings); - } finally { - repositoryLock.writeLock().unlock(); - } + public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { + attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of( + new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) + )).get(); - if (settings.getFetchPeriod() != 0) { - fetchTask.cancel(true); - fetchTask = scheduleFetch(settings.getFetchPeriod()); - } + clearRepository(tenantId); + initRepository(tenantId, settings); } + @SneakyThrows @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) .orElse(null); } @@ -387,7 +474,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); String entityId = StringUtils.substringBetween(path, "/", ".json"); return EntityIdFactory.getByTypeAndUuid(entityType, entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 435c661b02..03343ec6f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -20,19 +20,18 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; +import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; import java.util.List; public interface EntitiesVersionControlService { - EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; - - EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; @@ -42,23 +41,19 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; - - List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; - - List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - List listAllowedBranches(TenantId tenantId); + List listBranches(TenantId tenantId) throws Exception; - void saveSettings(EntitiesVersionControlSettings settings); + void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings); - EntitiesVersionControlSettings getSettings(); + EntitiesVersionControlSettings getSettings(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 06e0a9c7aa..602f5dba4f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -17,18 +17,10 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; -import java.util.List; -import java.util.Map; -import java.util.UUID; - @Data public class EntitiesVersionControlSettings { - private String repositoryDirectory; private String repositoryUri; private String username; private String password; - - private int fetchPeriod; - - private Map> tenantsAllowedBranches; + private String defaultBranch; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java new file mode 100644 index 0000000000..ac13a7b8bf --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; + +@Data +public class VersionCreationResult { + private EntityVersion version; + private int added; + private int modified; + private int removed; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java index 4cf0ee2b0f..cf8b588008 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java @@ -20,14 +20,14 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; @Data @AllArgsConstructor @NoArgsConstructor @Builder -public class EntityVersionLoadResult { - private ExportableEntity newEntityVersion; - private ExportableEntity previousEntityVersion; +public class VersionLoadResult { private EntityType entityType; + private int created; + private int updated; + private int deleted; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java similarity index 69% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java index d71ddc8720..835f684021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityFilterExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java @@ -13,26 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.query.EntityFilter; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class CustomEntityFilterExportRequest extends ExportRequest { +public class EntitiesByCustomFilterVersionCreateConfig extends VersionCreateConfig { private EntityFilter filter; - private int page; - private int pageSize; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.CUSTOM_ENTITY_FILTER; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.CUSTOM_ENTITY_FILTER; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java similarity index 72% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java index 946676cf65..6386e9299a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/CustomEntityQueryExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java @@ -13,24 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.query.EntityDataQuery; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class CustomEntityQueryExportRequest extends ExportRequest { +public class EntitiesByCustomQueryVersionCreateConfig extends VersionCreateConfig { private EntityDataQuery query; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.CUSTOM_ENTITY_QUERY; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.CUSTOM_ENTITY_QUERY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java index 01309a421b..1f3e1e376f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityListExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java @@ -13,23 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityId; import java.util.List; -@EqualsAndHashCode(callSuper = true) @Data -public class EntityListExportRequest extends ExportRequest { +public class EntityListVersionCreateConfig extends VersionCreateConfig { private List entitiesIds; @Override - public ExportRequestType getType() { - return ExportRequestType.ENTITY_LIST; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.ENTITY_LIST; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java index fd2f659b70..6a8d59b876 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/EntityTypeExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; @@ -21,18 +21,18 @@ import org.thingsboard.server.common.data.EntityType; import java.util.UUID; -@EqualsAndHashCode(callSuper = true) @Data -public class EntityTypeExportRequest extends ExportRequest { +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateConfig extends VersionCreateConfig { private EntityType entityType; - private int page; - private int pageSize; private UUID customerId; + private boolean removeOtherRemoteEntitiesOfType; + @Override - public ExportRequestType getType() { - return ExportRequestType.ENTITY_TYPE; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.ENTITY_TYPE; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java similarity index 77% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java index fe07ba7534..ab23f1175e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/SingleEntityExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java @@ -13,21 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.EntityId; -@EqualsAndHashCode(callSuper = true) @Data -public class SingleEntityExportRequest extends ExportRequest { +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateConfig extends VersionCreateConfig { private EntityId entityId; @Override - public ExportRequestType getType() { - return ExportRequestType.SINGLE_ENTITY; + public VersionCreateConfigType getType() { + return VersionCreateConfigType.SINGLE_ENTITY; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index af0f161227..31b7b46f47 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -23,17 +23,17 @@ import lombok.Data; @JsonTypeInfo(use = Id.NAME, property = "type") @JsonSubTypes({ - @Type(name = "SINGLE_ENTITY", value = SingleEntityExportRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListExportRequest.class), - @Type(name = "ENTITY_TYPE", value = EntityTypeExportRequest.class), - @Type(name = "CUSTOM_ENTITY_FILTER", value = CustomEntityFilterExportRequest.class), - @Type(name = "CUSTOM_ENTITY_QUERY", value = CustomEntityQueryExportRequest.class) + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateConfig.class), + @Type(name = "ENTITY_LIST", value = EntityListVersionCreateConfig.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateConfig.class), + @Type(name = "CUSTOM_ENTITY_FILTER", value = EntitiesByCustomFilterVersionCreateConfig.class), + @Type(name = "CUSTOM_ENTITY_QUERY", value = EntitiesByCustomQueryVersionCreateConfig.class) }) @Data -public abstract class ExportRequest { +public abstract class VersionCreateConfig { - private EntityExportSettings exportSettings; + private boolean saveRelations; - public abstract ExportRequestType getType(); + public abstract VersionCreateConfigType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java index c36ac567e3..b2fb251d6a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exporting/data/request/ExportRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exporting.data.request; +package org.thingsboard.server.service.sync.vc.data.request.create; -public enum ExportRequestType { +public enum VersionCreateConfigType { SINGLE_ENTITY, ENTITY_LIST, ENTITY_TYPE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 842f6ddd4e..47b07af21c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -13,12 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; +import java.util.List; + @Data -public class EntityVersionLoadSettings { - private boolean loadRelations; - private boolean findExistingEntityByName; +public class VersionCreateRequest { + + private String versionName; + private String branch; + + private List configs; + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java new file mode 100644 index 0000000000..f210dc9e0c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -0,0 +1,12 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { + + private boolean removeNonexistentLocalEntities; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java new file mode 100644 index 0000000000..6a49c45c2c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadRequest extends VersionLoadRequest { + + private Map configs; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java new file mode 100644 index 0000000000..f7d99a0981 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java @@ -0,0 +1,11 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; + +@Data +public class EntityVersionLoadConfig { + + private boolean loadRelations; + private boolean findExistingEntityByName; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java new file mode 100644 index 0000000000..f5336b55d1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionLoadRequest extends VersionLoadRequest { + + private EntityId externalEntityId; + + private EntityVersionLoadConfig config; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java index 1fc36463ba..660f5dc18d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java @@ -13,11 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @Data -public class EntityVersionSaveSettings { - private boolean saveRelations; +public abstract class VersionLoadRequest { + + private String branch; + private String versionId; + + public abstract VersionLoadRequestType getType(); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java new file mode 100644 index 0000000000..178d49668f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java @@ -0,0 +1,6 @@ +package org.thingsboard.server.service.sync.vc.data.request.load; + +public enum VersionLoadRequestType { + SINGLE_ENTITY, + ENTITY_TYPE +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 205bbca50e..14cdbf7b2c 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -24,6 +24,7 @@ import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.LogCommand; import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.Status; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; @@ -33,18 +34,18 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; -import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import java.io.File; import java.io.IOException; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { @@ -54,6 +55,8 @@ public class GitRepository { @Getter private final String directory; + @Getter + private final ReadWriteLock lock = new ReentrantReadWriteLock(); private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; @@ -61,21 +64,20 @@ public class GitRepository { this.directory = directory; } - public static GitRepository clone(String uri, String directory, - String username, String password) throws GitAPIException { + public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() .setURI(uri) - .setDirectory(new java.io.File(directory)) + .setDirectory(directory) .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new GitRepository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory.getAbsolutePath()); } - public static GitRepository open(String directory, String username, String password) throws IOException { - Git git = Git.open(new java.io.File(directory)); - return new GitRepository(git, newCredentialsProvider(username, password), directory); + public static GitRepository open(File directory, String username, String password) throws IOException { + Git git = Git.open(directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath()); } @@ -180,9 +182,17 @@ public class GitRepository { execute(git.clean()); } - - public Commit commit(String message, String filesPattern) throws GitAPIException { + public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); + } + + public Status status() throws GitAPIException { + org.eclipse.jgit.api.Status status = execute(git.status()); + return new Status(status.getAdded(), status.getModified(), status.getRemoved()); + } + + public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); @@ -256,4 +266,11 @@ public class GitRepository { private final String authorName; } + @Data + public static class Status { + private final Set added; + private final Set modified; + private final Set removed; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java index b5c5a5aab8..988bae7146 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntitiesExportImportControllerTest.java @@ -64,13 +64,13 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.ImportRequest; import java.nio.ByteBuffer; import java.util.Arrays; @@ -339,17 +339,17 @@ public abstract class BaseEntitiesExportImportControllerTest extends AbstractCon protected , I extends EntityId> EntityExportData exportSingleEntity(I entityId) throws Exception { - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(entityId); exportRequest.setExportSettings(new EntityExportSettings()); return (EntityExportData) exportEntities(exportRequest).get(0); } - protected List> exportEntities(ExportRequest exportRequest) throws Exception { + protected List> exportEntities(VersionCreateConfig exportRequest) throws Exception { return getResponse(doPost("/api/entities/export", exportRequest), new TypeReference>>() {}); } - protected List> exportEntities(List exportRequests) throws Exception { + protected List> exportEntities(List exportRequests) throws Exception { return getResponse(doPost("/api/entities/export?multiple", exportRequests), new TypeReference>>() {}); } diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java index bd237e39d0..faa7f3a318 100644 --- a/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntitiesExportImportControllerSqlTest.java @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.query.EntityListFilter; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; @@ -50,18 +49,18 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.ota.OtaPackageStateService; -import org.thingsboard.server.service.sync.exporting.data.DeviceExportData; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exporting.data.RuleChainExportData; -import org.thingsboard.server.service.sync.exporting.data.request.CustomEntityFilterExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.request.EntityListExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.EntityTypeExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.ExportRequest; -import org.thingsboard.server.service.sync.exporting.data.request.SingleEntityExportRequest; -import org.thingsboard.server.service.sync.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.importing.data.request.ImportRequest; +import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.exportimport.importing.data.ImportRequest; import java.util.ArrayList; import java.util.List; @@ -294,11 +293,11 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp dashboard.setConfiguration(dashboardConfiguration); dashboard = dashboardService.saveDashboard(dashboard); - EntityTypeExportRequest assetsExportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig assetsExportRequest = new EntityTypeVersionCreateConfig(); assetsExportRequest.setEntityType(EntityType.ASSET); assetsExportRequest.setPageSize(10); assetsExportRequest.setExportSettings(new EntityExportSettings()); - EntityTypeExportRequest dashboardsExportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig dashboardsExportRequest = new EntityTypeVersionCreateConfig(); dashboardsExportRequest.setEntityType(EntityType.DASHBOARD); dashboardsExportRequest.setPageSize(10); dashboardsExportRequest.setExportSettings(new EntityExportSettings()); @@ -365,7 +364,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Customer 1 - Device 1"); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), ruleChain.getId(), deviceProfile.getId(), dashboard.getId())); List> exportDataList = exportEntities(exportRequest); @@ -418,7 +417,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -464,7 +463,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation = createRelation(asset.getId(), device.getId()); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(asset.getId(), device.getId())); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -506,7 +505,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device1 = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(asset.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -538,7 +537,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Device device = createDevice(tenantId1, null, null, "Device 1"); EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); - SingleEntityExportRequest exportRequest = new SingleEntityExportRequest(); + SingleEntityVersionCreateConfig exportRequest = new SingleEntityVersionCreateConfig(); exportRequest.setEntityId(device.getId()); exportRequest.setExportSettings(EntityExportSettings.builder() .exportRelations(true) @@ -568,7 +567,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp logInAsTenantAdmin1(); DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(defaultDeviceProfile.getId())); exportRequest.setExportSettings(new EntityExportSettings()); List> exportDataList = exportEntities(exportRequest); @@ -613,7 +612,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } private void testEntityTypeExportRequest(ExportableEntity entity) throws Exception { - EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig exportRequest = new EntityTypeVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entity.getId().getEntityType()); @@ -626,11 +625,11 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp } private void testCustomEntityFilterExportRequest(ExportableEntity entity) throws Exception { - CustomEntityFilterExportRequest exportRequest = new CustomEntityFilterExportRequest(); + EntitiesByCustomFilterVersionCreateConfig exportRequest = new EntitiesByCustomFilterVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); - EntityListFilter filter = new EntityListFilter(); + org.thingsboard.server.common.data.query.EntityListFilter filter = new org.thingsboard.server.common.data.query.EntityListFilter(); filter.setEntityType(entity.getId().getEntityType()); filter.setEntityList(List.of(entity.getId().toString())); exportRequest.setFilter(filter); @@ -652,10 +651,10 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp Asset tenantAsset = createAsset(tenantId1, null, "A", "Tenant asset 1"); Asset customerAsset = createAsset(tenantId1, customer.getId(), "A", "Customer asset 1"); - List exportRequests = new ArrayList<>(); + List exportRequests = new ArrayList<>(); for (EntityType entityType : Set.of(EntityType.DEVICE, EntityType.ASSET)) { - EntityTypeExportRequest exportRequest = new EntityTypeExportRequest(); + EntityTypeVersionCreateConfig exportRequest = new EntityTypeVersionCreateConfig(); exportRequest.setExportSettings(new EntityExportSettings()); exportRequest.setPageSize(10); exportRequest.setEntityType(entityType); @@ -685,7 +684,7 @@ public class EntitiesExportImportControllerSqlTest extends BaseEntitiesExportImp DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); - EntityListExportRequest exportRequest = new EntityListExportRequest(); + EntityListVersionCreateConfig exportRequest = new EntityListVersionCreateConfig(); exportRequest.setEntitiesIds(List.of(customer.getId(), asset.getId(), device.getId(), ruleChain.getId(), dashboard.getId(), deviceProfile.getId())); exportRequest.setExportSettings(new EntityExportSettings()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 18444bb5f3..9240838543 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; public abstract class DaoUtil { @@ -51,7 +53,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap()); } - public static Pageable toPageable(PageLink pageLink, Map columnMap) { + public static Pageable toPageable(PageLink pageLink, Map columnMap) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(pageLink.getSortOrder(), columnMap)); } @@ -59,7 +61,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } - public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(sortOrders, columnMap)); } @@ -108,4 +110,18 @@ public abstract class DaoUtil { return ids; } + public static void processInBatches(Function> finder, int batchSize, Consumer processor) { + PageLink pageLink = new PageLink(batchSize); + PageData batch; + + boolean hasNextBatch; + do { + batch = finder.apply(pageLink); + batch.getData().forEach(processor); + + hasNextBatch = batch.hasNext(); + pageLink = pageLink.nextPageLink(); + } while (hasNextBatch); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java index cd534f24e6..aaa35e109a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -16,6 +16,8 @@ package org.thingsboard.server.dao; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import java.util.UUID; @@ -25,4 +27,6 @@ public interface ExportableEntityDao> extends Dao< default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } + PageData findByTenantId(UUID tenantId, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index bcfdaae965..8879809ddb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -219,6 +219,11 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im return findAssetsByTenantIdAndName(tenantId, name).orElse(null); } + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findAssetsByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.ASSET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index c6ba310793..034431157a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -80,6 +80,11 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findCustomersByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.CUSTOMER; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 4a870b2a21..847314f1a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -31,4 +33,6 @@ public interface DashboardRepository extends JpaRepository findByTenantIdAndTitle(UUID tenantId, String title); + Page findByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 6ff9fc6bfb..8fa0cbdc50 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -21,6 +21,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; @@ -58,6 +60,11 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.toPageData(dashboardRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); + } + @Override public List findByTenantIdAndTitle(UUID tenantId, String title) { return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 93cd857660..d4663f60a4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -313,6 +313,11 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao return findDeviceByTenantIdAndName(tenantId, name).orElse(null); } + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findDevicesByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index cae94cfb73..4cab209f49 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -121,6 +121,11 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findDeviceProfiles(TenantId.fromUUID(tenantId), pageLink); + } + @Override public EntityType getEntityType() { return EntityType.DEVICE_PROFILE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 0d5fea6f98..2703c99fba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -114,6 +114,11 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId, pageLink); + } + @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; From 2302213b3c8f288fa8284973e668285509f5e671 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 16 May 2022 17:03:23 +0300 Subject: [PATCH 038/156] Refactor version create request structures --- .../DefaultEntitiesVersionControlService.java | 81 ++++++++++--------- ...tiesByCustomFilterVersionCreateConfig.java | 36 --------- ...itiesByCustomQueryVersionCreateConfig.java | 36 --------- .../create/EntityListVersionCreateConfig.java | 33 -------- .../EntityListVersionCreateRequest.java | 21 +++++ .../create/EntityTypeVersionCreateConfig.java | 38 --------- .../EntityTypeVersionCreateRequest.java | 20 +++++ .../MultipleEntitiesVersionCreateConfig.java | 10 +++ .../SingleEntityVersionCreateConfig.java | 33 -------- .../SingleEntityVersionCreateRequest.java | 19 +++++ .../vc/data/request/create/SyncStrategy.java | 6 ++ .../request/create/VersionCreateConfig.java | 16 ---- .../create/VersionCreateConfigType.java | 25 ------ .../request/create/VersionCreateRequest.java | 6 +- .../create/VersionCreateRequestType.java | 7 ++ .../load/EntityTypeVersionLoadConfig.java | 2 +- 16 files changed, 127 insertions(+), 262 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 919960ce62..f6434daecc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -65,6 +65,7 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.request.create.MultipleEntitiesVersionCreateConfig; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; @@ -152,23 +153,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont if (repository.listBranches().contains(request.getBranch())) { repository.checkout(request.getBranch()); repository.merge(request.getBranch()); - } else { // FIXME [viacheslav]: rollback orphan branch on failure + } else { // TODO [viacheslav]: rollback orphan branch on failure repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch } - for (VersionCreateRequest.Config config : request.getConfigs()) { + for (VersionCreateConfig config : request.getConfigs()) { EntityExportSettings exportSettings = EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build(); List> entityDataList = new ArrayList<>(); - for (EntityId entityId : findEntities()) { + for (EntityId entityId : findEntities(user, config, 0, Integer.MAX_VALUE)) { // TODO [viacheslav]: find with pagination EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); entityDataList.add(entityData); } - if (config.isRemoveOtherRemoteEntitiesOfType()) { - entityDataList.stream() + if (config instanceof MultipleEntitiesVersionCreateConfig && + ((MultipleEntitiesVersionCreateConfig) config).isRemoveOtherRemoteEntitiesOfType()) { + entityDataList.stream() // FIXME [viacheslav]: in case of an emtpy entity type? none will be deleted on remote? .map(EntityExportData::getEntityType) .distinct() .forEach(entityType -> { @@ -186,7 +188,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); } } - // TODO [viacheslav]: find with pagination repository.add("."); @@ -206,18 +207,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private List findEntities(SecurityUser user, VersionCreateConfig entityFilter, int page, int pageSize) { - switch (entityFilter.getType()) { + private List findEntities(SecurityUser user, VersionCreateConfig config, int page, int pageSize) { + switch (config.getType()) { case SINGLE_ENTITY: { - SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) entityFilter; + SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) config; return List.of(filter.getEntityId()); } case ENTITY_LIST: { - EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) entityFilter; + EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) config; return filter.getEntitiesIds(); } case ENTITY_TYPE: { - EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) entityFilter; + EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) config; EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); @@ -228,7 +229,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return findEntities(user, newFilter, page, pageSize); } case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) entityFilter; + EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) config; EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); EntityDataPageLink pageLink = new EntityDataPageLink(); @@ -243,7 +244,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return findEntities(user, newFilter, page, pageSize); } case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) entityFilter; + EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) config; CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() .stream().map(EntityData::getEntityId) @@ -341,35 +342,35 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont boolean removeNonexistentLocalEntities = false; if () - if (request.isRemoveOtherLocalEntitiesOfType()) { - importResults.stream() - .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? - .forEach((entityType, resultsForEntityType) -> { - Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); - AtomicInteger deleted = new AtomicInteger(); - - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); + if (request.isRemoveOtherLocalEntitiesOfType()) { + importResults.stream() + .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? + .forEach((entityType, resultsForEntityType) -> { + Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); + AtomicInteger deleted = new AtomicInteger(); + + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + // need to delete in a specific order? + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + deleted.getAndIncrement(); } - // need to delete in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); - } + }); + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) + .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) + .deleted(deleted.get()) + .build()); }); - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) - .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) - .deleted(deleted.get()) - .build()); - }); - } + } return new ArrayList<>(results.values()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java deleted file mode 100644 index 835f684021..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomFilterVersionCreateConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.query.EntityFilter; - -import java.util.UUID; - -@Data -public class EntitiesByCustomFilterVersionCreateConfig extends VersionCreateConfig { - - private EntityFilter filter; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.CUSTOM_ENTITY_FILTER; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java deleted file mode 100644 index 6386e9299a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntitiesByCustomQueryVersionCreateConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.query.EntityDataQuery; - -import java.util.UUID; - -@Data -public class EntitiesByCustomQueryVersionCreateConfig extends VersionCreateConfig { - - private EntityDataQuery query; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.CUSTOM_ENTITY_QUERY; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java deleted file mode 100644 index 1f3e1e376f..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; - -@Data -public class EntityListVersionCreateConfig extends VersionCreateConfig { - - private List entitiesIds; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.ENTITY_LIST; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java new file mode 100644 index 0000000000..fadd9b1fd9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java @@ -0,0 +1,21 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityListVersionCreateRequest extends VersionCreateRequest { + + private List entitiesIds; + private MultipleEntitiesVersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.ENTITY_LIST; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java deleted file mode 100644 index 6a8d59b876..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.EntityType; - -import java.util.UUID; - -@Data -@EqualsAndHashCode(callSuper = true) -public class EntityTypeVersionCreateConfig extends VersionCreateConfig { - - private EntityType entityType; - private UUID customerId; - - private boolean removeOtherRemoteEntitiesOfType; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.ENTITY_TYPE; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java new file mode 100644 index 0000000000..0fd1d60667 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java @@ -0,0 +1,20 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateRequest extends VersionCreateRequest { + + private Map entityTypes; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.ENTITY_TYPE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java new file mode 100644 index 0000000000..f6d852875d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java @@ -0,0 +1,10 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public abstract class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { + private SyncStrategy syncStrategy; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java deleted file mode 100644 index ab23f1175e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; - -@Data -@EqualsAndHashCode(callSuper = true) -public class SingleEntityVersionCreateConfig extends VersionCreateConfig { - - private EntityId entityId; - - @Override - public VersionCreateConfigType getType() { - return VersionCreateConfigType.SINGLE_ENTITY; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java new file mode 100644 index 0000000000..31bdeca658 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java @@ -0,0 +1,19 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateRequest extends VersionCreateRequest { + + private EntityId entityId; + private VersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.SINGLE_ENTITY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java new file mode 100644 index 0000000000..6216ad7658 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java @@ -0,0 +1,6 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +public enum SyncStrategy { + MERGE, + OVERWRITE +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index 31b7b46f47..28f3232cb9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -15,25 +15,9 @@ */ package org.thingsboard.server.service.sync.vc.data.request.create; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonSubTypes.Type; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.annotation.JsonTypeInfo.Id; import lombok.Data; -@JsonTypeInfo(use = Id.NAME, property = "type") -@JsonSubTypes({ - @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateConfig.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateConfig.class), - @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateConfig.class), - @Type(name = "CUSTOM_ENTITY_FILTER", value = EntitiesByCustomFilterVersionCreateConfig.class), - @Type(name = "CUSTOM_ENTITY_QUERY", value = EntitiesByCustomQueryVersionCreateConfig.class) -}) @Data public abstract class VersionCreateConfig { - private boolean saveRelations; - - public abstract VersionCreateConfigType getType(); - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java deleted file mode 100644 index b2fb251d6a..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfigType.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -public enum VersionCreateConfigType { - SINGLE_ENTITY, - ENTITY_LIST, - ENTITY_TYPE, - - CUSTOM_ENTITY_FILTER, - CUSTOM_ENTITY_QUERY -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 47b07af21c..e11e2f85a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -17,14 +17,12 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; -import java.util.List; - @Data -public class VersionCreateRequest { +public abstract class VersionCreateRequest { private String versionName; private String branch; - private List configs; + public abstract VersionCreateRequestType getType(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java new file mode 100644 index 0000000000..a5d0447b40 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -0,0 +1,7 @@ +package org.thingsboard.server.service.sync.vc.data.request.create; + +public enum VersionCreateRequestType { + SINGLE_ENTITY, + ENTITY_LIST, + ENTITY_TYPE +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java index f210dc9e0c..b5ee5ddd50 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -7,6 +7,6 @@ import lombok.EqualsAndHashCode; @EqualsAndHashCode(callSuper = true) public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { - private boolean removeNonexistentLocalEntities; + private boolean removeOtherEntities; } From 5a43788b2498700135b4df4f6f1e5d7ed81b4f5c Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 17 May 2022 15:32:07 +0300 Subject: [PATCH 039/156] VC refactoring; Swagger examples --- .../EntitiesVersionControlController.java | 141 ++++++- .../DefaultEntitiesExportImportService.java | 40 +- .../EntitiesExportImportService.java | 5 + .../DefaultExportableEntitiesService.java | 46 --- .../exporting/ExportableEntitiesService.java | 2 - .../DefaultEntitiesVersionControlService.java | 386 +++++++++--------- .../sync/vc/data/VersionCreationResult.java | 15 + .../sync/vc/data/VersionedEntityInfo.java | 1 - .../EntityListVersionCreateRequest.java | 15 + .../EntityTypeVersionCreateRequest.java | 15 + .../MultipleEntitiesVersionCreateConfig.java | 17 +- .../SingleEntityVersionCreateRequest.java | 15 + .../vc/data/request/create/SyncStrategy.java | 15 + .../request/create/VersionCreateConfig.java | 2 +- .../request/create/VersionCreateRequest.java | 9 + .../create/VersionCreateRequestType.java | 15 + .../load/EntityTypeVersionLoadConfig.java | 17 +- .../load/EntityTypeVersionLoadRequest.java | 17 +- .../request/load/EntityVersionLoadConfig.java | 11 - .../load/SingleEntityVersionLoadRequest.java | 17 +- .../data/request/load/VersionLoadConfig.java | 26 ++ .../data/request/load/VersionLoadRequest.java | 9 + .../request/load/VersionLoadRequestType.java | 15 + .../server/utils/GitRepository.java | 4 +- 24 files changed, 580 insertions(+), 275 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 0625dbeb38..e7df5acdc3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; @@ -44,6 +45,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; + @RestController @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -53,6 +56,62 @@ public class EntitiesVersionControlController extends BaseController { private final EntitiesVersionControlService versionControlService; + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"saveRelations\": true\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_LIST:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_LIST\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entitiesIds\": [\n" + + " {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " {\n" + + " \"entityType\": \"DEVICE_PROFILE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " }\n" + + " ],\n" + + " \"config\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"MERGE\"\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"MERGE\"\n" + + " },\n" + + " \"DEVICE_PROFILE\": {\n" + + " \"saveRelations\": true,\n" + + " \"syncStrategy\": \"OVERWRITE\"\n" + + " }\n" + + " }\n" + + "}\n```") @PostMapping("/version") public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); @@ -64,6 +123,13 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profile 1 version 1.0\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") public List listEntityVersions(@PathVariable String branch, @PathVariable EntityType entityType, @@ -76,6 +142,13 @@ public class EntitiesVersionControlController extends BaseController { } } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profiles from dev\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}/{entityType}") public List listEntityTypeVersions(@PathVariable String branch, @PathVariable EntityType entityType) throws ThingsboardException { @@ -86,6 +159,21 @@ public class EntitiesVersionControlController extends BaseController { } } + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" + + " \"name\": \"Device 1 removed\"\n" + + " },\n" + + " {\n" + + " \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \"name\": \"Device profiles added\"\n" + + " },\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Devices added\"\n" + + " }\n" + + "]\n```") @GetMapping("/version/{branch}") public List listVersions(@PathVariable String branch) throws ThingsboardException { try { @@ -118,6 +206,38 @@ public class EntitiesVersionControlController extends BaseController { } + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + " \n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \n" + + " \"externalEntityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false,\n" + + " \"removeOtherEntities\": true\n" + + " }\n" + + " }\n" + + "}\n```") @PostMapping("/entity") public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); @@ -139,8 +259,21 @@ public class EntitiesVersionControlController extends BaseController { } - @ApiModelProperty(notes = "" + - "") + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"name\": \"master\",\n" + + " \"default\": true\n" + + " },\n" + + " {\n" + + " \"name\": \"dev\",\n" + + " \"default\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"dev-2\",\n" + + " \"default\": false\n" + + " }\n" + + "]\n\n```") @GetMapping("/branches") public List listBranches() throws ThingsboardException { try { @@ -161,7 +294,7 @@ public class EntitiesVersionControlController extends BaseController { } - @ApiModelProperty(notes = "" + + @ApiOperation(value = "", notes = "" + "```\n{\n" + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + " \"username\": \"User\",\n" + @@ -177,7 +310,7 @@ public class EntitiesVersionControlController extends BaseController { } } - @ApiModelProperty(notes = "" + + @ApiOperation(value = "", notes = "" + "```\n{\n" + " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + " \"username\": \"User\",\n" + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java index 608a256514..108c2c0b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java @@ -70,15 +70,37 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS } + @Override + public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + + EntityType entityType = exportData.getEntityType(); + EntityImportService> importService = getImportService(entityType); + + EntityImportResult importResult = importService.importEntity(user, exportData, importSettings); + + if (saveReferences) { + importResult.getSaveReferencesCallback().run(); + } + if (sendEvents) { + importResult.getSendEventsCallback().run(); + } + + return importResult; + } + @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixOrder(exportDataList); + fixDataOrderForImport(exportDataList); List> importResults = new ArrayList<>(); for (EntityExportData exportData : exportDataList) { - EntityImportResult importResult = importEntity(user, exportData, importSettings); + EntityImportResult importResult = importEntity(user, exportData, importSettings, false, false); importResults.add(importResult); } @@ -103,21 +125,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } - private void fixOrder(List> exportDataList) { + @Override + public void fixDataOrderForImport(List> exportDataList) { exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); } - private , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings) throws ThingsboardException { - if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { - throw new DataValidationException("Invalid entity data"); - } - - EntityType entityType = exportData.getEntityType(); - EntityImportService> importService = getImportService(entityType); - - return importService.importEntity(user, exportData, importSettings); - } - @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java index 5c29feb57c..049cf4390a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java @@ -30,6 +30,11 @@ public interface EntitiesExportImportService { , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, + boolean saveReferences, boolean sendEvents) throws ThingsboardException; + List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; + void fixDataOrderForImport(List> exportDataList); + } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java index 4d772ad935..dafd68ba7b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java @@ -17,11 +17,9 @@ package org.thingsboard.server.service.sync.exportimport.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.dao.DataAccessException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; @@ -47,12 +45,6 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; import java.util.Collection; import java.util.Collections; @@ -145,44 +137,6 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi } - @Transactional(readOnly = true, timeout = 40) - @Override - public List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig request) { - switch (request.getType()) { - case SINGLE_ENTITY: { - return List.of(((SingleEntityVersionCreateConfig) request).getEntityId()); - } - case ENTITY_LIST: { - return ((EntityListVersionCreateConfig) request).getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeVersionCreateConfig exportRequest = (EntityTypeVersionCreateConfig) request; - org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); - entityTypeFilter.setEntityType(exportRequest.getEntityType()); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, entityTypeFilter, 0, Integer.MAX_VALUE); - } - case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig exportRequest = (EntitiesByCustomFilterVersionCreateConfig) request; - EntityFilter filter = exportRequest.getFilter(); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByFilter(tenantId, customerId, filter, 0, Integer.MAX_VALUE); - } - case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig exportRequest = (EntitiesByCustomQueryVersionCreateConfig) request; - EntityDataQuery query = exportRequest.getQuery(); - - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(exportRequest.getCustomerId(), EntityId.NULL_UUID)); - return findEntitiesByQuery(tenantId, customerId, query); - } - default: { - throw new IllegalArgumentException("Export request is not supported"); - } - } - } - private List findEntitiesByFilter(TenantId tenantId, CustomerId customerId, EntityFilter filter, int page, int pageSize) { EntityDataPageLink pageLink = new EntityDataPageLink(); pageLink.setPage(page); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java index 043ec91e20..66c9079148 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java @@ -39,8 +39,6 @@ public interface ExportableEntitiesService { , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); - List findEntitiesByFilter(TenantId tenantId, VersionCreateConfig filter); // FIXME [viacheslav]: - void deleteByTenantIdAndId(TenantId tenantId, I id); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index f6434daecc..0bdedb5e4d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -18,16 +18,14 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -40,12 +38,12 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; @@ -65,17 +63,18 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.MultipleEntitiesVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadSettings; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomFilterVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntitiesByCustomQueryVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.EntityTypeVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; +import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.utils.GitRepository; +import org.thingsboard.server.utils.ThrowingRunnable; import java.io.File; import java.io.IOException; @@ -108,10 +107,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final AttributesService attributesService; private final EntityService entityService; private final TenantDao tenantDao; + private final TransactionTemplate transactionTemplate; // TODO [viacheslav]: concurrency private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}") + @Value("${java.io.tmpdir}/repositories") private String repositoriesFolder; private static final String SETTINGS_KEY = "vc"; @@ -146,33 +146,26 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { GitRepository repository = checkRepository(user.getTenantId()); - repository.getLock().writeLock().lock(); - - try { - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } - - for (VersionCreateConfig config : request.getConfigs()) { - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportRelations(config.isSaveRelations()) - .build(); - List> entityDataList = new ArrayList<>(); - for (EntityId entityId : findEntities(user, config, 0, Integer.MAX_VALUE)) { // TODO [viacheslav]: find with pagination - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); - entityDataList.add(entityData); - } + repository.fetch(); + if (repository.listBranches().contains(request.getBranch())) { + repository.checkout(request.getBranch()); + repository.merge(request.getBranch()); + } else { // TODO [viacheslav]: rollback orphan branch on failure + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } - if (config instanceof MultipleEntitiesVersionCreateConfig && - ((MultipleEntitiesVersionCreateConfig) config).isRemoveOtherRemoteEntitiesOfType()) { - entityDataList.stream() // FIXME [viacheslav]: in case of an emtpy entity type? none will be deleted on remote? - .map(EntityExportData::getEntityType) - .distinct() + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; + saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + break; + } + case ENTITY_LIST: { + EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; + if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { + versionCreateRequest.getEntitiesIds().stream() + .map(EntityId::getEntityType).distinct() .forEach(entityType -> { try { FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); @@ -181,76 +174,70 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } }); } - - for (EntityExportData entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { + saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); } + break; } + case ENTITY_TYPE: { + EntityTypeVersionCreateRequest versionCreateRequest = (EntityTypeVersionCreateRequest) request; + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { + try { + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); + entityDataPageLink.setPage(-1); + entityDataPageLink.setPageSize(-1); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + DaoUtil.processInBatches(pageLink -> { + entityDataPageLink.setPage(pageLink.getPage()); + entityDataPageLink.setPageSize(pageLink.getPageSize()); + return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); + }, 100, data -> { + EntityId entityId = data.getEntityId(); + try { + saveEntityData(user, repository, entityId, config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + }); + break; + } + } - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); + repository.add("."); - result.setVersion(toVersion(commit)); - return result; - } finally { - repository.getLock().writeLock().unlock(); - } - } + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); - private List findEntities(SecurityUser user, VersionCreateConfig config, int page, int pageSize) { - switch (config.getType()) { - case SINGLE_ENTITY: { - SingleEntityVersionCreateConfig filter = (SingleEntityVersionCreateConfig) config; - return List.of(filter.getEntityId()); - } - case ENTITY_LIST: { - EntityListVersionCreateConfig filter = (EntityListVersionCreateConfig) config; - return filter.getEntitiesIds(); - } - case ENTITY_TYPE: { - EntityTypeVersionCreateConfig filter = (EntityTypeVersionCreateConfig) config; - EntitiesByCustomFilterVersionCreateConfig newFilter = new EntitiesByCustomFilterVersionCreateConfig(); + GitRepository.Commit commit = repository.commit(request.getVersionName()); + repository.push(); - org.thingsboard.server.common.data.query.EntityTypeFilter entityTypeFilter = new org.thingsboard.server.common.data.query.EntityTypeFilter(); - entityTypeFilter.setEntityType(filter.getEntityType()); + result.setVersion(toVersion(commit)); + return result; + } - newFilter.setFilter(entityTypeFilter); - newFilter.setCustomerId(filter.getCustomerId()); - return findEntities(user, newFilter, page, pageSize); - } - case CUSTOM_ENTITY_FILTER: { - EntitiesByCustomFilterVersionCreateConfig filter = (EntitiesByCustomFilterVersionCreateConfig) config; - EntitiesByCustomQueryVersionCreateConfig newFilter = new EntitiesByCustomQueryVersionCreateConfig(); - - EntityDataPageLink pageLink = new EntityDataPageLink(); - pageLink.setPage(page); - pageLink.setPageSize(pageSize); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - pageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(filter.getFilter(), pageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - newFilter.setQuery(query); - newFilter.setCustomerId(filter.getCustomerId()); - return findEntities(user, newFilter, page, pageSize); - } - case CUSTOM_ENTITY_QUERY: { - EntitiesByCustomQueryVersionCreateConfig filter = (EntitiesByCustomQueryVersionCreateConfig) config; - CustomerId customerId = new CustomerId(ObjectUtils.defaultIfNull(filter.getCustomerId(), EntityId.NULL_UUID)); - return entityService.findEntityDataByQuery(user.getTenantId(), customerId, filter.getQuery()).getData() - .stream().map(EntityData::getEntityId) - .collect(Collectors.toList()); - } - } + private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .build()); + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); } @@ -271,14 +258,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private List listVersions(TenantId tenantId, String branch, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - } finally { - repository.getLock().readLock().unlock(); - } + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); } @@ -294,23 +276,15 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - EntityExportData entityData = getEntityDataAtVersion(tenantId, entityId, versionId); - - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - info.setEntityName(entityData.getEntity().getName()); - return info; - }) - .collect(Collectors.toList()); - } finally { - repository.getLock().readLock().unlock(); - } + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); } @@ -318,81 +292,117 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { GitRepository repository = checkRepository(user.getTenantId()); - List> entityDataList = new ArrayList<>(); - EntityVersion version; - repository.getLock().readLock().lock(); - try { - version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), path)) { - EntityExportData entityData = getEntityDataAtVersion(user.getTenantId(), info.getExternalId(), versionId); - entityDataList.add(entityData); - } - } finally { - repository.getLock().readLock().unlock(); - } - - EntityImportSettings importSettings = EntityImportSettings.builder() - .updateRelations(settings.isLoadRelations()) - .findExistingByName(settings.isFindExistingEntityByName()) - .build(); - // FIXME [viacheslav]: do evrth in transaction - List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); - - Map results = new HashMap<>(); + EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - boolean removeNonexistentLocalEntities = false; - if () - if (request.isRemoveOtherLocalEntitiesOfType()) { - importResults.stream() - .collect(Collectors.groupingBy(EntityImportResult::getEntityType)) // FIXME [viacheslav]: if no entities of entity type - remove all ? - .forEach((entityType, resultsForEntityType) -> { - Set modifiedEntities = resultsForEntityType.stream().map(EntityImportResult::getSavedEntity).map(ExportableEntity::getExternalId).collect(Collectors.toSet()); - AtomicInteger deleted = new AtomicInteger(); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + EntityImportResult importResult = transactionTemplate.execute(status -> { + try { + EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + result.getSaveReferencesCallback().run(); + return result; + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + try { + importResult.getSendEventsCallback().run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + return List.of(VersionLoadResult.builder() + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } + case ENTITY_TYPE: { + EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; + return transactionTemplate.execute(status -> { + Map results = new HashMap<>(); + List saveReferencesCallbacks = new ArrayList<>(); + List sendEventsCallbacks = new ArrayList<>(); + // order entity types.. + // or what + versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + AtomicInteger deleted = new AtomicInteger(); + + Set remoteEntities; + try { + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + .map(VersionedEntityInfo::getExternalId) + .collect(Collectors.toSet()); + for (EntityId externalEntityId : remoteEntities) { + EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + if (config.isRemoveOtherEntities()) { DaoUtil.processInBatches(pageLink -> { return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); }, 100, entity -> { - if (entity.getExternalId() == null || !modifiedEntities.contains(entity.getExternalId())) { + if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { try { exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); } catch (ThingsboardException e) { throw new RuntimeException(e); } - // need to delete in a specific order? + // need to delete entity types in a specific order? exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); deleted.getAndIncrement(); } }); - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() == null).count()) - .updated((int) resultsForEntityType.stream().filter(importResult -> importResult.getOldEntity() != null).count()) - .deleted(deleted.get()) - .build()); - }); + } + + results.put(entityType, VersionLoadResult.builder() + .created(created.get()) + .updated(updated.get()) + .deleted(deleted.get()) + .build()); + }); + + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { + try { + saveReferencesCallback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + } + return new ArrayList<>(results.values()); + }); } - - return new ArrayList<>(results.values()); - } - - @SneakyThrows - private EntityExportData getEntityDataAtVersion(TenantId tenantId, EntityId externalId, String versionId) { - GitRepository repository = checkRepository(tenantId); - repository.getLock().readLock().lock(); - try { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); - } finally { - repository.getLock().readLock().unlock(); + default: + throw new IllegalArgumentException("Unsupported version load request"); } } - private void updateEntityVersionInfo(TenantId tenantId, EntityId entityId, EntityVersion version) { - ObjectNode versionInfo = JacksonUtil.newObjectNode(); - versionInfo.set("versionName", new TextNode(version.getName())); - versionInfo.set("versionId", new TextNode(version.getId())); - attributesService.save(tenantId, entityId, DataConstants.SERVER_SCOPE, - List.of(new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry("entityVersionInfo", versionInfo.toString())))); + private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); + + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); } @@ -417,12 +427,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; - if (Files.exists(repositoryDirectory)) { - repository = GitRepository.open(repositoryDirectory.toFile(), settings.getUsername(), settings.getPassword()); - } else { - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - } + FileUtils.forceDelete(repositoryDirectory.toFile()); + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); repositories.put(tenantId, repository); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java index ac13a7b8bf..9194c67272 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 97bb6ada12..0a65297892 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -21,6 +21,5 @@ import org.thingsboard.server.common.data.id.EntityId; @Data public class VersionedEntityInfo { private EntityId externalId; - private String entityName; // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java index fadd9b1fd9..59cf7d0176 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java index 0fd1d60667..f53e563e28 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java index f6d852875d..0a38e3c922 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/MultipleEntitiesVersionCreateConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; @@ -5,6 +20,6 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) -public abstract class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { +public class MultipleEntitiesVersionCreateConfig extends VersionCreateConfig { private SyncStrategy syncStrategy; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java index 31bdeca658..4d03404199 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java index 6216ad7658..43ec873f22 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; public enum SyncStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java index 28f3232cb9..d4f8354df4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java @@ -18,6 +18,6 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; @Data -public abstract class VersionCreateConfig { +public class VersionCreateConfig { private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index e11e2f85a3..f4ba122f6f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -15,8 +15,17 @@ */ package org.thingsboard.server.service.sync.vc.data.request.create; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), + @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionCreateRequest.class) +}) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java index a5d0447b40..add3591fa2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.create; public enum VersionCreateRequestType { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java index b5ee5ddd50..27597f5e87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -5,7 +20,7 @@ import lombok.EqualsAndHashCode; @Data @EqualsAndHashCode(callSuper = true) -public class EntityTypeVersionLoadConfig extends EntityVersionLoadConfig { +public class EntityTypeVersionLoadConfig extends VersionLoadConfig { private boolean removeOtherEntities; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java index 6a49c45c2c..0ce902c83e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -10,7 +25,7 @@ import java.util.Map; @EqualsAndHashCode(callSuper = true) public class EntityTypeVersionLoadRequest extends VersionLoadRequest { - private Map configs; + private Map entityTypes; @Override public VersionLoadRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java deleted file mode 100644 index f7d99a0981..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityVersionLoadConfig.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.thingsboard.server.service.sync.vc.data.request.load; - -import lombok.Data; - -@Data -public class EntityVersionLoadConfig { - - private boolean loadRelations; - private boolean findExistingEntityByName; - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java index f5336b55d1..eb4bb1bdc1 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; import lombok.Data; @@ -10,7 +25,7 @@ public class SingleEntityVersionLoadRequest extends VersionLoadRequest { private EntityId externalEntityId; - private EntityVersionLoadConfig config; + private VersionLoadConfig config; @Override public VersionLoadRequestType getType() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java new file mode 100644 index 0000000000..6ead76f105 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data.request.load; + +import lombok.Data; + +@Data +public class VersionLoadConfig { + + private boolean loadRelations; + private boolean findExistingEntityByName; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java index 660f5dc18d..5a24bc78e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java @@ -15,8 +15,17 @@ */ package org.thingsboard.server.service.sync.vc.data.request.load; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; import lombok.Data; +import static com.fasterxml.jackson.annotation.JsonSubTypes.Type; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionLoadRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionLoadRequest.class) +}) @Data public abstract class VersionLoadRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java index 178d49668f..6a59cba676 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.service.sync.vc.data.request.load; public enum VersionLoadRequestType { diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 14cdbf7b2c..2cb01e6a8b 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -55,8 +55,6 @@ public class GitRepository { @Getter private final String directory; - @Getter - private final ReadWriteLock lock = new ReentrantReadWriteLock(); private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; @@ -88,7 +86,7 @@ public class GitRepository { public void checkout(String branch) throws GitAPIException { execute(git.checkout() - .setName(branch)); + .setName("origin/" + branch)); } public void merge(String branch) throws IOException, GitAPIException { From 9b209e407553509821ce8b3ad010d6624f3a1ea6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 17 May 2022 18:00:53 +0300 Subject: [PATCH 040/156] Refactor version create request; fixes for git api usage --- .../EntitiesVersionControlController.java | 45 +++------ .../DefaultEntitiesVersionControlService.java | 96 ++++++++++--------- .../EntityListVersionCreateRequest.java | 36 ------- .../create/EntityTypeVersionCreateConfig.java | 4 +- .../request/create/VersionCreateRequest.java | 3 +- .../create/VersionCreateRequestType.java | 1 - .../server/utils/GitRepository.java | 30 +++--- 7 files changed, 86 insertions(+), 129 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index e7df5acdc3..7c2c2ac483 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -72,43 +71,27 @@ public class EntitiesVersionControlController extends BaseController { " \"saveRelations\": true\n" + " }\n" + "}\n```" + NEW_LINE + - "ENTITY_LIST:" + NEW_LINE + + "COMPLEX:" + NEW_LINE + "```\n{\n" + - " \"type\": \"ENTITY_LIST\",\n" + + " \"type\": \"COMPLEX\",\n" + "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + - "\n" + - " \"entitiesIds\": [\n" + - " {\n" + - " \"entityType\": \"DEVICE\",\n" + - " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + - " },\n" + - " {\n" + - " \"entityType\": \"DEVICE_PROFILE\",\n" + - " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + - " }\n" + - " ],\n" + - " \"config\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + - " }\n" + - "}\n```" + NEW_LINE + - "ENTITY_TYPE:" + NEW_LINE + - "```\n{\n" + - " \"type\": \"ENTITY_TYPE\",\n" + - "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + + " \"versionName\": \"Devices and profiles: release 2\",\n" + + " \"branch\": \"master\",\n" + "\n" + + " \"syncStrategy\": \"OVERWRITE\",\n" + " \"entityTypes\": {\n" + " \"DEVICE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + + " \"syncStrategy\": null,\n" + + " \"allEntities\": true,\n" + + " \"saveRelations\": true\n" + " },\n" + " \"DEVICE_PROFILE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"OVERWRITE\"\n" + + " \"syncStrategy\": \"MERGE\",\n" + + " \"allEntities\": false,\n" + + " \"entityIds\": [\n" + + " \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " ],\n" + + " \"saveRelations\": true\n" + " }\n" + " }\n" + "}\n```") diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 8919f7d46e..7fa99d9e6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -22,7 +22,10 @@ import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; @@ -63,7 +66,6 @@ import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; @@ -88,6 +90,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -149,10 +152,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont repository.fetch(); if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); + repository.checkout("origin/" + request.getBranch(), false); + try { + repository.checkout(request.getBranch(), true); + } catch (RefAlreadyExistsException e) { + repository.checkout(request.getBranch(), false); + } repository.merge(request.getBranch()); } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + try { + repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } } switch (request.getType()) { @@ -161,28 +175,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } - case ENTITY_LIST: { - EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; - if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { - versionCreateRequest.getEntitiesIds().stream() - .map(EntityId::getEntityType).distinct() - .forEach(entityType -> { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { - saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); - } - break; - } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { try { FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); } catch (IOException e) { @@ -190,27 +186,38 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); - try { - saveEntityData(user, repository, entityId, config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); + entityTypeFilter.setEntityType(entityType); + EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); + entityDataPageLink.setPage(-1); + entityDataPageLink.setPageSize(-1); + EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); + entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); + EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); + + DaoUtil.processInBatches(pageLink -> { + entityDataPageLink.setPage(pageLink.getPage()); + entityDataPageLink.setPageSize(pageLink.getPageSize()); + return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); + }, 100, data -> { + EntityId entityId = data.getEntityId(); + try { + saveEntityData(user, repository, entityId, config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + } catch (Exception e) { + throw new RuntimeException(e); + } } - }); + } + }); break; } @@ -427,7 +434,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; - FileUtils.forceDelete(repositoryDirectory.toFile()); + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } Files.createDirectories(repositoryDirectory); repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); @@ -450,7 +459,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - clearRepository(tenantId); initRepository(tenantId, settings); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java deleted file mode 100644 index adeac70e89..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc.data.request.create; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; - -import java.util.List; - -@Data -@EqualsAndHashCode(callSuper = true) -public class EntityListVersionCreateRequest extends VersionCreateRequest { - - private List entitiesIds; - private EntityTypeVersionCreateConfig config; - - @Override - public VersionCreateRequestType getType() { - return VersionCreateRequestType.ENTITY_LIST; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java index 1c27d9d2f1..0cc890c0d7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java @@ -17,9 +17,9 @@ package org.thingsboard.server.service.sync.vc.data.request.create; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; import java.util.List; +import java.util.UUID; @Data @EqualsAndHashCode(callSuper = true) @@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig { //optional private SyncStrategy syncStrategy; - private List entityIds; + private List entityIds; private boolean allEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java index 10f093d26d..a99bd56d7f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java @@ -23,8 +23,7 @@ import lombok.Data; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), - @Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class) + @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class) }) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java index fea8b6fec7..aacc57f3b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java @@ -17,6 +17,5 @@ package org.thingsboard.server.service.sync.vc.data.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, - ENTITY_LIST, COMPLEX } diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 2cb01e6a8b..ac8df955d6 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -23,8 +23,7 @@ import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.GitCommand; import org.eclipse.jgit.api.ListBranchCommand; import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.RmCommand; -import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; @@ -44,8 +43,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { @@ -84,9 +81,15 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch) throws GitAPIException { + public void checkout(String branch, boolean createBranch) throws GitAPIException { execute(git.checkout() - .setName("origin/" + branch)); + .setCreateBranch(createBranch) + .setName(branch)); + } + + public void reset() throws GitAPIException { + execute(git.reset() + .setMode(ResetCommand.ResetType.HARD)); } public void merge(String branch) throws IOException, GitAPIException { @@ -170,14 +173,15 @@ public class GitRepository { public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) + .setForced(true) .setName(name)); - Set uncommittedChanges = git.status().call().getUncommittedChanges(); - if (!uncommittedChanges.isEmpty()) { - RmCommand rm = git.rm(); - uncommittedChanges.forEach(rm::addFilepattern); - execute(rm); - } - execute(git.clean()); +// Set uncommittedChanges = git.status().call().getUncommittedChanges(); +// if (!uncommittedChanges.isEmpty()) { +// RmCommand rm = git.rm(); +// uncommittedChanges.forEach(rm::addFilepattern); +// execute(rm); +// } +// execute(git.clean()); } public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] From 7ec45fd5f1d602f23fde6225b7c4ee7b77e66f52 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 10:40:02 +0300 Subject: [PATCH 041/156] Rename Data structures. Create separate module. --- application/pom.xml | 4 + .../server/controller/AssetController.java | 4 +- .../server/controller/DeviceController.java | 4 +- .../server/controller/EdgeController.java | 4 +- .../EntitiesVersionControlController.java | 14 +-- .../service/asset/AssetBulkImportService.java | 4 +- .../device/DeviceBulkImportService.java | 4 +- .../service/edge/EdgeBulkImportService.java | 4 +- .../DefaultEntitiesExportImportService.java | 20 ++-- .../EntitiesExportImportService.java | 10 +- .../DefaultExportableEntitiesService.java | 2 +- .../exporting/EntityExportService.java | 6 +- .../exporting/ExportableEntitiesService.java | 5 +- .../impl/BaseEntityExportService.java | 6 +- .../impl/DefaultEntityExportService.java | 10 +- .../exporting/impl/DeviceExportService.java | 4 +- .../impl/RuleChainExportService.java | 4 +- .../importing/EntityImportService.java | 8 +- .../csv/AbstractBulkImportService.java | 3 +- .../importing/csv/BulkImportColumnType.java | 2 +- .../importing/csv/BulkImportRequest.java | 2 +- .../importing/csv/BulkImportResult.java | 2 +- .../importing/csv/ImportedEntityInfo.java | 2 +- .../importing/impl/AssetImportService.java | 4 +- .../impl/BaseEntityImportService.java | 12 +-- .../importing/impl/CustomerImportService.java | 4 +- .../impl/DashboardImportService.java | 6 +- .../importing/impl/DeviceImportService.java | 4 +- .../impl/DeviceProfileImportService.java | 4 +- .../impl/RuleChainImportService.java | 6 +- .../DefaultEntitiesVersionControlService.java | 42 ++++----- .../vc/EntitiesVersionControlService.java | 14 +-- .../common/data/sync}/JsonTbEntity.java | 2 +- .../common/data/sync}/ThrowingRunnable.java | 2 +- .../data/sync/ie}/DeviceExportData.java | 2 +- .../data/sync/ie}/EntityExportData.java | 4 +- .../data/sync/ie}/EntityExportSettings.java | 2 +- .../data/sync/ie}/EntityImportResult.java | 6 +- .../data/sync/ie}/EntityImportSettings.java | 2 +- .../data/sync/ie}/RuleChainExportData.java | 2 +- .../vc}/EntitiesVersionControlSettings.java | 2 +- .../common/data/sync/vc}/EntityVersion.java | 2 +- .../data/sync/vc}/VersionCreationResult.java | 2 +- .../data/sync/vc}/VersionLoadResult.java | 2 +- .../data/sync/vc}/VersionedEntityInfo.java | 2 +- .../create/ComplexVersionCreateRequest.java | 2 +- .../create/EntityTypeVersionCreateConfig.java | 2 +- .../SingleEntityVersionCreateRequest.java | 2 +- .../sync/vc}/request/create/SyncStrategy.java | 2 +- .../request/create/VersionCreateConfig.java | 2 +- .../request/create/VersionCreateRequest.java | 2 +- .../create/VersionCreateRequestType.java | 2 +- .../load/EntityTypeVersionLoadConfig.java | 2 +- .../load/EntityTypeVersionLoadRequest.java | 2 +- .../load/SingleEntityVersionLoadRequest.java | 2 +- .../vc}/request/load/VersionLoadConfig.java | 2 +- .../vc}/request/load/VersionLoadRequest.java | 2 +- .../request/load/VersionLoadRequestType.java | 2 +- common/pom.xml | 1 + common/version-control/pom.xml | 94 +++++++++++++++++++ pom.xml | 5 + 61 files changed, 241 insertions(+), 141 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/DefaultEntitiesExportImportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/EntitiesExportImportService.java (80%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/DefaultExportableEntitiesService.java (99%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/EntityExportService.java (81%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/ExportableEntitiesService.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/BaseEntityExportService.java (86%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/DefaultEntityExportService.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/DeviceExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/exporting/impl/RuleChainExportService.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/EntityImportService.java (78%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/AbstractBulkImportService.java (99%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportColumnType.java (96%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportRequest.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/BulkImportResult.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/csv/ImportedEntityInfo.java (91%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/AssetImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/BaseEntityImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/CustomerImportService.java (93%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DashboardImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DeviceImportService.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/DeviceProfileImportService.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/{exportimport => ie}/importing/impl/RuleChainImportService.java (95%) rename {application/src/main/java/org/thingsboard/server/utils => common/data/src/main/java/org/thingsboard/server/common/data/sync}/JsonTbEntity.java (97%) rename {application/src/main/java/org/thingsboard/server/utils => common/data/src/main/java/org/thingsboard/server/common/data/sync}/ThrowingRunnable.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/DeviceExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityExportSettings.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityImportResult.java (89%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/EntityImportSettings.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/ie}/RuleChainExportData.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/EntitiesVersionControlSettings.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/EntityVersion.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionCreationResult.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionLoadResult.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/VersionedEntityInfo.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/ComplexVersionCreateRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/EntityTypeVersionCreateConfig.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/SingleEntityVersionCreateRequest.java (93%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/SyncStrategy.java (90%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateConfig.java (91%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/create/VersionCreateRequestType.java (91%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/EntityTypeVersionLoadConfig.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/EntityTypeVersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/SingleEntityVersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadConfig.java (92%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadRequest.java (94%) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/sync/vc}/request/load/VersionLoadRequestType.java (91%) create mode 100644 common/version-control/pom.xml diff --git a/application/pom.xml b/application/pom.xml index 211d22fb04..2ff2204b5a 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -69,6 +69,10 @@ org.thingsboard.common cluster-api + + org.thingsboard.common + version-control + org.thingsboard.rule-engine rule-engine-components diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 01401deebe..2a3268146f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -51,8 +51,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 61e44241d7..8b6f49c491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -66,8 +66,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8136b6d052..b3bbb03f4c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -52,8 +52,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 7c2c2ac483..2ab75c565d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -32,13 +32,13 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index a2b71267b4..5eaeec0b73 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -26,8 +26,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index ee119c4dce..f0a7b6231e 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index fea50538cc..b0fb2161e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -28,8 +28,8 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 108c2c0b1b..72e3d757be 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,15 +27,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 049cf4390a..5f5bed9dac 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index dafd68ba7b..18c5f5c7c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 5532f30125..9b251a3dab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 66c9079148..742b993cfc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -25,9 +25,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; - -import java.util.List; public interface ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 46428a763f..80850cb462 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 69af613480..31bf768e90 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 93f9c51655..2e03b92a44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index f40ee6884a..15c9c72a74 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java index be6f62efe4..bf249937b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing; +package org.thingsboard.server.service.sync.ie.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 2be56ebe2a..b92e100401 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -66,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java index 9c0e6b6491..24b566e631 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java index 6347910cab..e8eac6a9ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java index 550d4a72f3..0626c8e690 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java index a39dcc7481..d48e9a3d23 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 80a942c73f..fe6c3f1875 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7ecfa7d3ab..203b0caa16 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index b69c5a0d5b..2a7570e644 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index b85e403f87..16ebdd580e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 112152365b..eb123a63af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java index f97950aaf8..592fe4bc5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 993f9fa27a..1f1f15a74e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7fa99d9e6d..c22eb6a5a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -55,28 +55,28 @@ import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.utils.GitRepository; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; import java.io.IOException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 03343ec6f5..e025f80fed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,13 +19,13 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index cfc38ff3c3..9d326246d4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; import com.fasterxml.jackson.annotation.JsonSubTypes; diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java index 858ccdf422..1cb2ac8c74 100644 --- a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import org.thingsboard.server.common.data.exception.ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index 4be338d7cc..faf9280b93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 81e3b344d0..b361a302fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.utils.JsonTbEntity; +import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0fb318b5fb..051745c07f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index 242856bfcb..ea69425a2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.utils.JsonTbEntity; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.JsonTbEntity; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 075a6896ef..3e18bc9f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java index e1d870473f..b176782f62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 602f5dba4f..ee8f5b40b0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 3547f89e3c..8779d64ed3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java index 9194c67272..8cb09b1b30 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index cf8b588008..0039546fb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 0a65297892..163fe4c6d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java index 3c34680f95..f8d7279f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java index 0cc890c0d7..92cab96354 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java index 4d03404199..507afd116e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java index 43ec873f22..baf24efeb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum SyncStrategy { MERGE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index d4f8354df4..a1aa6cd6d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java index a99bd56d7f..9a23838921 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java index aacc57f3b3..d57363cf7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 27597f5e87..837986f82d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java index 0ce902c83e..f2cb83f0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java index eb4bb1bdc1..cf31317b4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 6ead76f105..31b34dbcb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java index 5a24bc78e5..d9d1329c8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java index 6a59cba676..190e6871d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; public enum VersionLoadRequestType { SINGLE_ENTITY, diff --git a/common/pom.xml b/common/pom.xml index 236fb50a19..62bb46e81c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,7 @@ cache coap-server edge-api + version-control diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml new file mode 100644 index 0000000000..1b7194e029 --- /dev/null +++ b/common/version-control/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + common + + org.thingsboard.common + version-control + jar + + Thingsboard Server Version Control API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.springframework + spring-core + + + com.google.guava + guava + provided + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.thingsboard.common + data + + + + + + + + + diff --git a/pom.xml b/pom.xml index e215e0719d..324de28f3a 100755 --- a/pom.xml +++ b/pom.xml @@ -866,6 +866,11 @@ util ${project.version} + + org.thingsboard.common + version-control + ${project.version} + org.thingsboard.common cache From c6ebf049a2dd7910293a3d728533686ec6bc663a Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 14:46:54 +0300 Subject: [PATCH 042/156] Version Control Service refactoring --- application/pom.xml | 4 - .../EntitiesVersionControlController.java | 2 +- .../DefaultEntitiesVersionControlService.java | 211 ++------------ .../vc/EntitiesVersionControlService.java | 2 +- .../vc/LocalGitVersionControlService.java | 258 ++++++++++++++++++ .../src/main/resources/thingsboard.yml | 5 + common/version-control/pom.xml | 21 ++ .../sync/vc/DefaultGitRepositoryService.java | 239 ++++++++++++++++ .../service/sync/vc}/GitRepository.java | 2 +- .../service/sync/vc/GitRepositoryService.java | 48 ++++ .../sync/vc/GitVersionControlService.java | 60 ++++ .../server/service/sync/vc/PendingCommit.java | 36 +++ 12 files changed, 695 insertions(+), 193 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java rename {application/src/main/java/org/thingsboard/server/utils => common/version-control/src/main/java/org/thingsboard/server/service/sync/vc}/GitRepository.java (99%) create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java diff --git a/application/pom.xml b/application/pom.xml index 2ff2204b5a..37d782b3b0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,10 +345,6 @@ Java-WebSocket test - - org.eclipse.jgit - org.eclipse.jgit - diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 2ab75c565d..2794db90d4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -172,7 +172,7 @@ public class EntitiesVersionControlController extends BaseController { @PathVariable EntityType entityType, @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c22eb6a5a5..e222d027e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -75,7 +75,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersion import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.utils.GitRepository; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; @@ -105,85 +104,31 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final GitVersionControlService gitService; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AttributesService attributesService; private final EntityService entityService; - private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; - // TODO [viacheslav]: concurrency - private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}/repositories") - private String repositoriesFolder; - - private static final String SETTINGS_KEY = "vc"; - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); - if (settings != null) { - try { - initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, 5, 5, TimeUnit.SECONDS); - } - + public static final String SETTINGS_KEY = "vc"; @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout("origin/" + request.getBranch(), false); - try { - repository.checkout(request.getBranch(), true); - } catch (RefAlreadyExistsException e) { - repository.checkout(request.getBranch(), false); - } - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } - } + + var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } + gitService.deleteAll(commit, entityType); } if (config.isAllEntities()) { @@ -203,7 +148,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, 100, data -> { EntityId entityId = data.getEntityId(); try { - saveEntityData(user, repository, entityId, config); + saveEntityData(user, commit, entityId, config); } catch (Exception e) { throw new RuntimeException(e); } @@ -211,7 +156,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -223,90 +168,50 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); - - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); - - result.setVersion(toVersion(commit)); - return result; + return gitService.push(commit); } - private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + gitService.addToCommit(commit, entityData); } @Override public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + return gitService.listVersions(tenantId, branch, externalId); } @Override public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + return gitService.listVersions(tenantId, branch, entityType); } @Override public List listVersions(TenantId tenantId, String branch) throws Exception { - return listVersions(tenantId, branch, null); - } - - private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return gitService.listVersions(tenantId, branch); } - @Override - public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, null); + return gitService.listEntitiesAtVersion(tenantId, branch, versionId); } - private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - return info; - }) - .collect(Collectors.toList()); - } - - @Override public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); result.getSaveReferencesCallback().run(); return result; } catch (Exception e) { @@ -340,11 +245,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Set remoteEntities; try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() .map(VersionedEntityInfo::getExternalId) .collect(Collectors.toSet()); for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); if (importResult.getOldEntity() == null) created.incrementAndGet(); else updated.incrementAndGet(); @@ -402,10 +307,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); - + private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .findExistingByName(config.isFindExistingEntityByName()) @@ -415,43 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listBranches(TenantId tenantId) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listBranches(); - } - - - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - - private GitRepository checkRepository(TenantId tenantId) { - return Optional.ofNullable(repositories.get(tenantId)) - .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); - } - - private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { - Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - if (Files.exists(repositoryDirectory)) { - FileUtils.forceDelete(repositoryDirectory.toFile()); - } - - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - repositories.put(tenantId, repository); - } - - private void clearRepository(TenantId tenantId) throws IOException { - GitRepository repository = repositories.get(tenantId); - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repositories.remove(tenantId); - } + return gitService.listBranches(tenantId); } - @SneakyThrows @Override public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { @@ -459,41 +328,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - initRepository(tenantId, settings); + gitService.initRepository(tenantId, settings); } - @SneakyThrows @Override public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - - private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; + return gitService.getSettings(tenantId); } - - private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); - String entityId = StringUtils.substringBetween(path, "/", ".json"); - return EntityIdFactory.getByTypeAndUuid(entityType, entityId); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index e025f80fed..f0d4169b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -41,7 +41,7 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java new file mode 100644 index 0000000000..b9a5a9260b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -0,0 +1,258 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.io.IOException; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; + +@Slf4j +@RequiredArgsConstructor +@Service +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +public class LocalGitVersionControlService implements GitVersionControlService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final GitRepositoryService gitRepositoryService; + private final TenantDao tenantDao; + private final AttributesService attributesService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + @AfterStartUp + public void init() { + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } + } + }); + } + + @Override + @SneakyThrows + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) + .orElse(null); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + + @Override + public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + var pendingCommit = new PendingCommit(tenantId, request); + PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); + if (old != null) { + gitRepositoryService.abort(old); + } + gitRepositoryService.prepareCommit(pendingCommit); + return pendingCommit; + } finally { + lock.unlock(); + } + } + + @Override + public void deleteAll(PendingCommit commit, EntityType entityType) { + doInsideLock(commit, c -> { + try { + gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public void addToCommit(PendingCommit commit, EntityExportData> entityData) { + doInsideLock(commit, c -> { + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString()), entityDataJson); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + return executeInsideLock(commit, gitRepositoryService::push); + } + + @Override + public List listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, branch, (String) null); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + try { + return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + @Override + public List listBranches(TenantId tenantId) { + return gitRepositoryService.listBranches(tenantId); + } + + @Override + public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { + try { + String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, + getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private List listVersions(TenantId tenantId, String branch, String path) { + try { + return gitRepositoryService.listVersions(tenantId, branch, path); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private void doInsideLock(PendingCommit commit, Consumer r) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + r.accept(commit); + } finally { + lock.unlock(); + } + } + + private T executeInsideLock(PendingCommit commit, Function c) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + return c.apply(commit); + } finally { + lock.unlock(); + } + } + + private void checkCommit(PendingCommit commit) { + PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); + if (existing == null || !existing.getTxId().equals(commit.getTxId())) { + throw new ConcurrentModificationException(); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03dbb49763..654929f500 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1115,6 +1115,11 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + git: + service: "${JS_VC_GIT_SERVICE:local}" # local/remote + repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + management: endpoints: web: diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 1b7194e029..f29b7db81a 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -40,6 +40,23 @@ org.springframework spring-core + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + com.google.guava guava @@ -65,6 +82,10 @@ ch.qos.logback logback-classic + + org.eclipse.jgit + org.eclipse.jgit + org.springframework.boot spring-boot-starter-test diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..d22fcf5d94 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,239 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + @Value("${vc.git.repos-poll-interval:60}") + private long reposPollInterval; + + private ScheduledExecutorService scheduler; + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + scheduler = Executors.newSingleThreadScheduledExecutor(); + scheduler.scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getRequest().getBranch(); + try { + repository.fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout("origin/" + branch, false); + try { + repository.checkout(branch, true); + } catch (RefAlreadyExistsException e) { + repository.checkout(branch, false); + } + repository.merge(branch); + } else { // TODO [viacheslav]: rollback orphan branch on failure + try { + repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); + repository.push(); + + result.setVersion(toVersion(gitCommit)); + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void abort(PendingCommit commit) { + //TODO: implement; + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + } + + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + } + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/utils/GitRepository.java rename to common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index ac8df955d6..afebe1f7fc 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..2191f38930 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.io.IOException; +import java.util.List; + +public interface GitRepositoryService { + + void prepareCommit(PendingCommit pendingCommit); + + List listVersions(TenantId tenantId, String branch, String path) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java new file mode 100644 index 0000000000..e67a8cca5f --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlService { + + @SneakyThrows + EntitiesVersionControlSettings getSettings(TenantId tenantId); + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); + + void addToCommit(PendingCommit commit, EntityExportData> entityData); + + void deleteAll(PendingCommit pendingCommit, EntityType entityType); + + VersionCreationResult push(PendingCommit commit); + + List listVersions(TenantId tenantId, String branch); + + List listVersions(TenantId tenantId, String branch, EntityType entityType); + + List listVersions(TenantId tenantId, String branch, EntityId entityId); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + List listBranches(TenantId tenantId); + + EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java new file mode 100644 index 0000000000..a30f7514a2 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +@Data +public class PendingCommit { + + private final UUID txId; + private final TenantId tenantId; + private final VersionCreateRequest request; + + public PendingCommit(TenantId tenantId, VersionCreateRequest request) { + this.txId = UUID.randomUUID(); + this.tenantId = tenantId; + this.request = request; + } +} From 1d6b9a5cbdc05c514787e6c6d795dc66549fcb73 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 18 May 2022 16:43:01 +0300 Subject: [PATCH 043/156] Version control executor --- common/version-control/pom.xml | 27 ++++ .../sync/vc/DefaultGitRepositoryService.java | 1 - .../version-control/src/main/proto/vc.proto | 71 +++++++++ msa/pom.xml | 1 + msa/vc-executor/pom.xml | 136 ++++++++++++++++++ msa/vc-executor/src/main/conf/logback.xml | 43 ++++++ .../src/main/conf/tb-vc-executor.conf | 22 +++ ...oardVersionControlExecutorApplication.java | 47 ++++++ .../src/main/resources/logback.xml | 34 +++++ .../src/main/resources/tb-vc-executor.yml | 27 ++++ 10 files changed, 408 insertions(+), 1 deletion(-) create mode 100644 common/version-control/src/main/proto/vc.proto create mode 100644 msa/vc-executor/pom.xml create mode 100644 msa/vc-executor/src/main/conf/logback.xml create mode 100644 msa/vc-executor/src/main/conf/tb-vc-executor.conf create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java create mode 100644 msa/vc-executor/src/main/resources/logback.xml create mode 100644 msa/vc-executor/src/main/resources/tb-vc-executor.yml diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index f29b7db81a..164640fe73 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -86,6 +86,21 @@ org.eclipse.jgit org.eclipse.jgit + + io.grpc + grpc-netty-shaded + provided + + + io.grpc + grpc-protobuf + provided + + + io.grpc + grpc-stub + provided + org.springframework.boot spring-boot-starter-test @@ -109,7 +124,19 @@ + + org.xolstice.maven.plugins + protobuf-maven-plugin + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index d22fcf5d94..c93b4ae76d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -226,7 +226,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } } - private EntityVersion toVersion(GitRepository.Commit commit) { return new EntityVersion(commit.getId(), commit.getMessage()); } diff --git a/common/version-control/src/main/proto/vc.proto b/common/version-control/src/main/proto/vc.proto new file mode 100644 index 0000000000..e8c7e09384 --- /dev/null +++ b/common/version-control/src/main/proto/vc.proto @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto3"; + +option java_package = "org.thingsboard.server.gen.vc.v1"; +option java_multiple_files = true; +option java_outer_classname = "EdgeProtos"; + +package vc; + +// Interface exported by the ThingsBoard Core. +service TbGitRpcService { + + rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} + +} + + +/** + * Data Structures; + */ +message CommitRequestMsg { + string txId = 1; + PrepareMsg prepareMsg = 2; + AddMsg addMsg = 3; + DeleteMsg deleteMsg = 4; + PushMsg pushMsg = 5; + AbortMsg abortMsg = 6; +} + +message CommitResponseMsg { + string id = 1; + string name = 2; + int32 added = 3; + int32 modified = 4; + int32 removed = 5; +} + +message PrepareMsg { + string tenantId = 1; + string commitMsg = 2; + string branchName = 3; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} \ No newline at end of file diff --git a/msa/pom.xml b/msa/pom.xml index 106f2be2a3..888291502b 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -40,6 +40,7 @@ tb + vc-executor js-executor web-ui tb-node diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml new file mode 100644 index 0000000000..7e4df36117 --- /dev/null +++ b/msa/vc-executor/pom.xml @@ -0,0 +1,136 @@ + + + 4.0.0 + + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor + + ThingsBoard Version Control Executor + https://thingsboard.io + Project for ThingsBoard version control microservice + + + UTF-8 + ${basedir}/../.. + java + false + process-resources + package + tb-mqtt-transport + false + ${project.build.directory}/windows + ThingsBoard Version Control Executor Service + org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication + + + + + org.thingsboard.common + version-control + + + org.springframework.boot + spring-boot-starter-web + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + + + diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml new file mode 100644 index 0000000000..d62cf2b3f5 --- /dev/null +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -0,0 +1,43 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + diff --git a/msa/vc-executor/src/main/conf/tb-vc-executor.conf b/msa/vc-executor/src/main/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..83287286bb --- /dev/null +++ b/msa/vc-executor/src/main/conf/tb-vc-executor.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java new file mode 100644 index 0000000000..2724a566ee --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -0,0 +1,47 @@ +package org.thingsboard.server.vc; /** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.sync.vc"}) +public class ThingsboardVersionControlExecutorApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "tb-vc-executor"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardVersionControlExecutorApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml new file mode 100644 index 0000000000..2834934ee0 --- /dev/null +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -0,0 +1,34 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + \ No newline at end of file diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml new file mode 100644 index 0000000000..1de47e2750 --- /dev/null +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# If you enabled process metrics you should also enable 'web-environment'. +spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" +# If you enabled process metrics you should set 'web-application-type' to 'servlet' value. +spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" + +server: + # Server bind address (has no effect if web-environment is disabled). + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port (has no effect if web-environment is disabled). + port: "${HTTP_BIND_PORT:8080}" + From 567ee5d75bb20b9a2803adbd506b5a8e23dfc450 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 18 May 2022 18:49:21 +0300 Subject: [PATCH 044/156] Refactor loadEntitiesVersion --- .../DefaultEntitiesExportImportService.java | 12 +- .../sync/ie/EntitiesExportImportService.java | 7 +- .../DefaultEntitiesVersionControlService.java | 175 +++++++----------- .../vc/LocalGitVersionControlService.java | 9 + .../sync/vc/GitVersionControlService.java | 5 +- 5 files changed, 95 insertions(+), 113 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 72e3d757be..38d3b00213 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -95,7 +95,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixDataOrderForImport(exportDataList); + exportDataList.sort(getDataComparatorForImport()); List> importResults = new ArrayList<>(); @@ -125,9 +125,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } + + @Override + public Comparator> getDataComparatorForImport() { + return Comparator.comparing(EntityExportData::getEntityType, getEntityTypeComparatorForImport()); + } + @Override - public void fixDataOrderForImport(List> exportDataList) { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + public Comparator getEntityTypeComparatorForImport() { + return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 5f5bed9dac..7bdb271444 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.ie; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -24,6 +25,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import java.util.Comparator; import java.util.List; public interface EntitiesExportImportService { @@ -35,6 +37,9 @@ public interface EntitiesExportImportService { List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; - void fixDataOrderForImport(List> exportDataList); + + Comparator> getDataComparatorForImport(); + + Comparator getEntityTypeComparatorForImport(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index e222d027e7..f6319db9c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,18 +15,10 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; @@ -34,24 +26,15 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -77,27 +60,16 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.ThrowingRunnable; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; - @Service @TbCoreComponent @RequiredArgsConstructor @@ -115,7 +87,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { @@ -132,23 +103,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } if (config.isAllEntities()) { - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { try { - saveEntityData(user, commit, entityId, config); + saveEntityData(user, commit, entity.getId(), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -209,22 +168,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + VersionLoadConfig config = versionLoadRequest.getConfig(); EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); - result.getSaveReferencesCallback().run(); - return result; + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); } }); - try { - importResult.getSendEventsCallback().run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); - } - return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) .created(importResult.getOldEntity() == null ? 1 : 0) .updated(importResult.getOldEntity() != null ? 1 : 0) .deleted(0) @@ -234,56 +191,68 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; return transactionTemplate.execute(status -> { Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); List sendEventsCallbacks = new ArrayList<>(); - // order entity types.. - // or what - versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); - AtomicInteger deleted = new AtomicInteger(); - - Set remoteEntities; - try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() - .map(VersionedEntityInfo::getExternalId) - .collect(Collectors.toSet()); - for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - if (config.isRemoveOtherEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - // need to delete entity types in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); + versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()) + .forEach(entityType -> { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + + try { + int limit = 100; + int offset = 0; + List> entityDataList; + do { + entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + for (EntityExportData entityData : entityDataList) { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + } catch (Exception e) { + throw new RuntimeException(e); } + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); }); - } - results.put(entityType, VersionLoadResult.builder() - .created(created.get()) - .updated(updated.get()) - .deleted(deleted.get()) - .build()); - }); + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + VersionLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); + } + }); + }); for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { try { @@ -307,14 +276,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { - EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - } - @Override public List listBranches(TenantId tenantId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index b9a5a9260b..cd96f5a6bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -52,6 +52,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; @Slf4j @RequiredArgsConstructor @@ -193,6 +194,14 @@ public class LocalGitVersionControlService implements GitVersionControlService { return gitRepositoryService.listBranches(tenantId); } + @Override + public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { + return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() + .skip(offset).limit(limit) + .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) + .collect(Collectors.toList()); + } + @Override public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { try { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index e67a8cca5f..35d68347b1 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.sync.vc; -import lombok.SneakyThrows; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @@ -31,7 +30,6 @@ import java.util.List; public interface GitVersionControlService { - @SneakyThrows EntitiesVersionControlSettings getSettings(TenantId tenantId); void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); @@ -57,4 +55,7 @@ public interface GitVersionControlService { List listBranches(TenantId tenantId); EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); + + List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); + } From 0b6de71768c6ff356224c177b222d5b50a1e66d1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 12:01:06 +0300 Subject: [PATCH 045/156] Implement version control ssh access with private key. Improve VC settings REST methods. Add tenantId field to admin settings. Add DB upgrade. --- application/pom.xml | 4 + .../main/data/upgrade/3.3.4/schema_update.sql | 3 + .../server/controller/AdminController.java | 61 +++++++- .../EntitiesVersionControlController.java | 47 +----- .../install/ThingsboardInstallService.java | 1 + .../fetch/AdminSettingsEdgeEventFetcher.java | 7 +- .../DefaultSystemDataLoaderService.java | 2 + .../install/SqlDatabaseUpgradeService.java | 12 ++ .../permission/TenantAdminPermissions.java | 1 + .../system/DefaultSystemSecurityService.java | 1 + .../DefaultEntitiesVersionControlService.java | 145 ++++++++++-------- .../vc/EntitiesVersionControlService.java | 12 +- .../server/utils/GitRepository.java | 131 +++++++++++++--- .../server/common/data/AdminSettings.java | 19 ++- .../server/common/data/StringUtils.java | 42 +++++ .../vc}/EntitiesVersionControlSettings.java | 6 +- .../data/vc/VersionControlAuthMethod.java | 21 +++ .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/AdminSettingsEntity.java | 9 ++ .../server/dao/settings/AdminSettingsDao.java | 4 +- .../settings/AdminSettingsServiceImpl.java | 22 ++- .../sql/settings/AdminSettingsRepository.java | 3 +- .../dao/sql/settings/JpaAdminSettingsDao.java | 4 +- .../main/resources/sql/schema-entities.sql | 1 + pom.xml | 5 + 25 files changed, 411 insertions(+), 154 deletions(-) rename {application/src/main/java/org/thingsboard/server/service/sync/vc/data => common/data/src/main/java/org/thingsboard/server/common/data/vc}/EntitiesVersionControlSettings.java (79%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java diff --git a/application/pom.xml b/application/pom.xml index 211d22fb04..83d36c2b42 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,6 +345,10 @@ org.eclipse.jgit org.eclipse.jgit + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index ad307a6576..f537acfb36 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -26,3 +26,6 @@ ALTER TABLE dashboard ADD COLUMN IF NOT EXISTS external_id UUID; ALTER TABLE customer ADD COLUMN IF NOT EXISTS external_id UUID; + +ALTER TABLE admin_settings + ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index e5934cbf26..3868476491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -20,12 +20,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; @@ -34,14 +29,17 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.update.UpdateService; import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @@ -60,6 +58,9 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Autowired + private EntitiesVersionControlService versionControlService; + @Autowired private UpdateService updateService; @@ -96,6 +97,7 @@ public class AdminController extends BaseController { @RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + adminSettings.setTenantId(getTenantId()); adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings)); if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); @@ -180,6 +182,53 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get version control settings (getVersionControlSettings)", + notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings") + @ResponseBody + public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", + notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/vsSettings") + public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + versionControlService.saveVersionControlSettings(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", + notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) + public void checkVersionControlAccess( + @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") + @RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + settings = checkNotNull(settings); + versionControlService.checkVersionControlAccess(getTenantId(), settings); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index e7df5acdc3..599149402f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,25 +15,18 @@ */ package org.thingsboard.server.controller; -import io.swagger.annotations.ApiModelProperty; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; @@ -280,7 +273,7 @@ public class EntitiesVersionControlController extends BaseController { List remoteBranches = versionControlService.listBranches(getTenantId()); List infos = new ArrayList<>(); - String defaultBranch = getSettings().getDefaultBranch(); + String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch(); if (StringUtils.isNotEmpty(defaultBranch)) { remoteBranches.remove(defaultBranch); infos.add(new BranchInfo(defaultBranch, true)); @@ -293,40 +286,6 @@ public class EntitiesVersionControlController extends BaseController { } } - - @ApiOperation(value = "", notes = "" + - "```\n{\n" + - " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + - " \"username\": \"User\",\n" + - " \"password\": \"api_key\",\n" + - " \"defaultBranch\": \"master\"\n" + - "}\n```") - @GetMapping("/settings") - public EntitiesVersionControlSettings getSettings() throws ThingsboardException { - try { - return versionControlService.getSettings(getTenantId()); - } catch (Exception e) { - throw handleException(e); - } - } - - @ApiOperation(value = "", notes = "" + - "```\n{\n" + - " \"repositoryUri\": \"https://github.com/User/repo.git\",\n" + - " \"username\": \"User\",\n" + - " \"password\": \"api_key\",\n" + - " \"defaultBranch\": \"master\"\n" + - "}\n```") - @PostMapping("/settings") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { - try { - versionControlService.saveSettings(getTenantId(), settings); - } catch (Exception e) { - throw handleException(e); - } - } - - @Data public static class BranchInfo { private final String name; diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index ebab403149..e728c004c0 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -219,6 +219,7 @@ public class ThingsboardInstallService { databaseEntitiesUpgradeService.upgradeDatabase("3.3.3"); case "3.3.4": log.info("Upgrading ThingsBoard from version 3.3.4 to 3.4.0 ..."); + databaseEntitiesUpgradeService.upgradeDatabase("3.3.4"); log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java index 0f4ecf7e30..ee10c7859b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java @@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings))); - AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); + AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings))); @@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates))); - AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); + AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates))); @@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { } } - private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) { + private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) { AdminSettings tenantMailSettings = new AdminSettings(); + tenantMailSettings.setTenantId(tenantId); jsonValue.put("useSystemMailSettings", true); tenantMailSettings.setJsonValue(jsonValue); tenantMailSettings.setKey(key); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 9e7636879f..696ac50c3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -231,6 +231,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override public void createAdminSettings() throws Exception { AdminSettings generalSettings = new AdminSettings(); + generalSettings.setTenantId(TenantId.SYS_TENANT_ID); generalSettings.setKey("general"); ObjectNode node = objectMapper.createObjectNode(); node.put("baseUrl", "http://localhost:8080"); @@ -239,6 +240,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); AdminSettings mailSettings = new AdminSettings(); + mailSettings.setTenantId(TenantId.SYS_TENANT_ID); mailSettings.setKey("mail"); node = objectMapper.createObjectNode(); node.put("mailFrom", "ThingsBoard "); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index f7b9e5cad1..1f9dcf15a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -534,6 +534,18 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; + case "3.3.4": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Updating schema ..."); + schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); + loadSql(schemaUpdateFile, conn); + log.info("Updating schema settings..."); + conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); + log.info("Schema updated."); + } catch (Exception e) { + log.error("Failed updating schema!!!", e); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java index 4ff884f0e1..8f821f5051 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java @@ -28,6 +28,7 @@ public class TenantAdminPermissions extends AbstractPermissions { public TenantAdminPermissions() { super(); + put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker); put(Resource.ALARM, tenantEntityPermissionChecker); put(Resource.ASSET, tenantEntityPermissionChecker); put(Resource.DEVICE, tenantEntityPermissionChecker); diff --git a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java index 9a998544f3..2449d7fbc0 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java @@ -107,6 +107,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService { AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, "securitySettings"); if (adminSettings == null) { adminSettings = new AdminSettings(); + adminSettings.setTenantId(tenantId); adminSettings.setKey("securitySettings"); } adminSettings.setJsonValue(JacksonUtil.valueToTree(securitySettings)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 8919f7d46e..77a5586a8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -19,34 +19,29 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.JsonDataEntry; -import org.thingsboard.server.common.data.kv.KvEntry; -import org.thingsboard.server.common.data.query.EntityDataPageLink; -import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.common.data.query.EntityDataSortOrder; -import org.thingsboard.server.common.data.query.EntityKey; -import org.thingsboard.server.common.data.query.EntityKeyType; -import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.*; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -58,17 +53,11 @@ import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExp import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.EntityListVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.ComplexVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SingleEntityVersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.create.SyncStrategy; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.create.*; import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; @@ -81,13 +70,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -104,7 +87,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; - private final AttributesService attributesService; + private final AdminSettingsService adminSettingsService; private final EntityService entityService; private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; @@ -114,14 +97,14 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Value("${java.io.tmpdir}/repositories") private String repositoriesFolder; - private static final String SETTINGS_KEY = "vc"; + private static final String SETTINGS_KEY = "entitiesVersionControl"; private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); + EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); if (settings != null) { try { initRepository(tenantId, settings); @@ -412,6 +395,73 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return repository.listBranches(); } + @Override + public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + } + + @Override + public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { + EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); + AdminSettings adminSettings = new AdminSettings(); + adminSettings.setTenantId(tenantId); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + EntitiesVersionControlSettings savedVersionControlSettings; + try { + savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + try { + clearRepository(tenantId); + initRepository(tenantId, savedVersionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } + return savedVersionControlSettings; + } + + @Override + public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { + EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + settings = this.restoreCredentials(settings, storedSettings); + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + try { + GitRepository.test(settings, repositoryDirectory.toFile()); + } catch (GitAPIException e) { + throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), + ThingsboardErrorCode.GENERAL); + } + } + + private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + if (storedSettings != null) { + settings.setPassword(storedSettings.getPassword()); + } + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + if (storedSettings != null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) && + StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { return listVersions(tenantId, branch, null).stream() @@ -426,11 +476,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - FileUtils.forceDelete(repositoryDirectory.toFile()); + FileUtils.forceDelete(repositoryDirectory.toFile()); Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + + GitRepository repository = GitRepository.clone(settings, repositoryDirectory.toFile()); + repositories.put(tenantId, repository); } @@ -442,34 +493,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - - @SneakyThrows - @Override - public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { - attributesService.save(tenantId, tenantId, DataConstants.SERVER_SCOPE, List.of( - new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) - )).get(); - - clearRepository(tenantId); - initRepository(tenantId, settings); - } - - @SneakyThrows - @Override - public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - private EntityVersion toVersion(GitRepository.Commit commit) { return new EntityVersion(commit.getId(), commit.getMessage()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 03343ec6f5..2854ec1d05 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,16 +16,17 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vc.data.EntityVersion; import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; +import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; import java.util.List; @@ -51,9 +52,12 @@ public interface EntitiesVersionControlService { List listBranches(TenantId tenantId) throws Exception; + EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); + + EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; - void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings); - EntitiesVersionControlSettings getSettings(TenantId tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 2cb01e6a8b..e207c30fa2 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -18,14 +18,8 @@ package org.thingsboard.server.utils; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; -import org.apache.commons.lang3.StringUtils; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.RmCommand; -import org.eclipse.jgit.api.Status; -import org.eclipse.jgit.api.TransportCommand; +import org.apache.sshd.common.util.security.SecurityUtils; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -34,50 +28,88 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.sshd.JGitKeyCache; +import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; +import org.eclipse.jgit.transport.sshd.SshdSessionFactory; +import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PublicKey; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; + private final SshdSessionFactory sshSessionFactory; @Getter private final String directory; - private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; + this.sshSessionFactory = sshSessionFactory; this.directory = directory; } - public static GitRepository clone(String uri, String username, String password, File directory) throws GitAPIException { - CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); - Git git = Git.cloneRepository() - .setURI(uri) + public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + CloneCommand cloneCommand = Git.cloneRepository() + .setURI(settings.getRepositoryUri()) .setDirectory(directory) - .setNoCheckout(true) - .setCredentialsProvider(credentialsProvider) - .call(); - return new GitRepository(git, credentialsProvider, directory.getAbsolutePath()); + .setNoCheckout(true); + configureTransportCommand(cloneCommand, credentialsProvider, sshSessionFactory); + Git git = cloneCommand.call(); + return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static GitRepository open(File directory, String username, String password) throws IOException { + public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException { Git git = Git.open(directory); - return new GitRepository(git, newCredentialsProvider(username, password), directory.getAbsolutePath()); + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + return new GitRepository(git, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } + public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); + configureTransportCommand(lsRemoteCommand, credentialsProvider, sshSessionFactory); + lsRemoteCommand.call(); + } public void fetch() throws GitAPIException { execute(git.fetch() @@ -108,7 +140,6 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { return listCommits(branch, null, limit); } @@ -246,16 +277,66 @@ public class GitRepository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand && credentialsProvider != null) { - ((TransportCommand) command).setCredentialsProvider(credentialsProvider); + if (command instanceof TransportCommand) { + configureTransportCommand((TransportCommand) command, credentialsProvider, sshSessionFactory); } return command.call(); } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { + if (credentialsProvider != null) { + transportCommand.setCredentialsProvider(credentialsProvider); + } + if (sshSessionFactory != null) { + transportCommand.setTransportConfigCallback(transport -> { + if (transport instanceof SshTransport) { + SshTransport sshTransport = (SshTransport) transport; + sshTransport.setSshSessionFactory(sshSessionFactory); + } + }); + } + } + private static CredentialsProvider newCredentialsProvider(String username, String password) { - return new UsernamePasswordCredentialsProvider(username, password); + return new UsernamePasswordCredentialsProvider(username, password == null ? "" : password); } + private static SshdSessionFactory newSshdSessionFactory(String privateKey, String password, File directory) { + SshdSessionFactory sshSessionFactory = null; + if (StringUtils.isNotBlank(privateKey)) { + Iterable keyPairs = loadKeyPairs(privateKey, password); + sshSessionFactory = new SshdSessionFactoryBuilder() + .setPreferredAuthentications("publickey") + .setDefaultKeysProvider(file -> keyPairs) + .setHomeDirectory(directory) + .setSshDirectory(directory) + .setServerKeyDatabase((file, file2) -> new ServerKeyDatabase() { + @Override + public List lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) { + return Collections.emptyList(); + } + + @Override + public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) { + return true; + } + }) + .build(new JGitKeyCache()); + } + return sshSessionFactory; + } + + private static Iterable loadKeyPairs(String privateKeyContent, String password) { + Iterable keyPairs = null; + try { + keyPairs = SecurityUtils.loadKeyPairIdentities(null, + null, new ByteArrayInputStream(privateKeyContent.getBytes()), (session, resourceKey, retryIndex) -> password); + } catch (Exception e) {} + if (keyPairs == null) { + throw new IllegalArgumentException("Failed to load ssh private key"); + } + return keyPairs; + } @Data public static class Commit { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index 3b639659c8..d88fabc052 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.AdminSettingsId; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class AdminSettings extends BaseData { +public class AdminSettings extends BaseData implements HasTenantId { private static final long serialVersionUID = -7670322981725511892L; + private TenantId tenantId; + @NoXss @Length(fieldName = "key") private String key; @@ -44,6 +47,7 @@ public class AdminSettings extends BaseData { public AdminSettings(AdminSettings adminSettings) { super(adminSettings); + this.tenantId = adminSettings.getTenantId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -60,7 +64,16 @@ public class AdminSettings extends BaseData { return super.getCreatedTime(); } - @ApiModelProperty(position = 3, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") + @ApiModelProperty(position = 3, value = "JSON object with Tenant Id.", readOnly = true) + public TenantId getTenantId() { + return tenantId; + } + + public void setTenantId(TenantId tenantId) { + this.tenantId = tenantId; + } + + @ApiModelProperty(position = 4, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") public String getKey() { return key; } @@ -69,7 +82,7 @@ public class AdminSettings extends BaseData { this.key = key; } - @ApiModelProperty(position = 4, value = "JSON representation of the Administration Settings value") + @ApiModelProperty(position = 5, value = "JSON representation of the Administration Settings value") public JsonNode getJsonValue() { return jsonValue; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index f1391555b0..52ad469ce1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -17,6 +17,10 @@ package org.thingsboard.server.common.data; public class StringUtils { + public static final String EMPTY = ""; + + public static final int INDEX_NOT_FOUND = -1; + public static boolean isEmpty(String source) { return source == null || source.isEmpty(); } @@ -32,4 +36,42 @@ public class StringUtils { public static boolean isNotBlank(String source) { return source != null && !source.isEmpty() && !source.trim().isEmpty(); } + + public static String removeStart(final String str, final String remove) { + if (isEmpty(str) || isEmpty(remove)) { + return str; + } + if (str.startsWith(remove)){ + return str.substring(remove.length()); + } + return str; + } + + public static String substringBefore(final String str, final String separator) { + if (isEmpty(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringBetween(final String str, final String open, final String close) { + if (str == null || open == null || close == null) { + return null; + } + final int start = str.indexOf(open); + if (start != INDEX_NOT_FOUND) { + final int end = str.indexOf(close, start + open.length()); + if (end != INDEX_NOT_FOUND) { + return str.substring(start + open.length(), end); + } + } + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java similarity index 79% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java index 602f5dba4f..ce4f8e71d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java @@ -13,14 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.vc; import lombok.Data; @Data public class EntitiesVersionControlSettings { private String repositoryUri; + private VersionControlAuthMethod authMethod; private String username; private String password; + private String privateKeyFileName; + private String privateKey; + private String privateKeyPassword; private String defaultBranch; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java new file mode 100644 index 0000000000..9d2dfa048e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.vc; + +public enum VersionControlAuthMethod { + USERNAME_PASSWORD, + PRIVATE_KEY +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 0b74e79f2b..2bb9ef6bdb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -90,6 +90,8 @@ public class ModelConstants { * Cassandra admin_settings constants. */ public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings"; + + public static final String ADMIN_SETTINGS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key"; public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value"; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 49d5002af2..4da17d2c34 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -22,14 +22,18 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; +import java.util.UUID; + import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_JSON_VALUE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY_PROPERTY; @@ -41,6 +45,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY @Table(name = ADMIN_SETTINGS_COLUMN_FAMILY_NAME) public final class AdminSettingsEntity extends BaseSqlEntity implements BaseEntity { + @Column(name = ModelConstants.ADMIN_SETTINGS_TENANT_ID_PROPERTY) + private UUID tenantId; + @Column(name = ADMIN_SETTINGS_KEY_PROPERTY) private String key; @@ -57,6 +64,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl this.setUuid(adminSettings.getId().getId()); } this.setCreatedTime(adminSettings.getCreatedTime()); + this.tenantId = adminSettings.getTenantId().getId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -65,6 +73,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettings toData() { AdminSettings adminSettings = new AdminSettings(new AdminSettingsId(id)); adminSettings.setCreatedTime(createdTime); + adminSettings.setTenantId(TenantId.fromUUID(tenantId)); adminSettings.setKey(key); adminSettings.setJsonValue(jsonValue); return adminSettings; diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index faea2cc34e..14455cddd4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.UUID; + public interface AdminSettingsDao extends Dao { /** @@ -35,6 +37,6 @@ public interface AdminSettingsDao extends Dao { * @param key the key * @return the admin settings object */ - AdminSettings findByKey(TenantId tenantId, String key); + AdminSettings findByTenantIdAndKey(UUID tenantId, String key); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bf4c71d296..0462cc38de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -46,20 +47,35 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings findAdminSettingsByKey(TenantId tenantId, String key) { log.trace("Executing findAdminSettingsByKey [{}]", key); Validator.validateString(key, "Incorrect key " + key); - return adminSettingsDao.findByKey(tenantId, key); + return adminSettingsDao.findByTenantIdAndKey(tenantId.getId(), key); } @Override public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if(adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { + if (adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); } + } else if (adminSettings.getKey().equals("entitiesVersionControl")) { + VersionControlAuthMethod authMethod = VersionControlAuthMethod.valueOf(adminSettings.getJsonValue().get("authMethod").asText()); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && !adminSettings.getJsonValue().has("password")) { + AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); + if (vcSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("password", vcSettings.getJsonValue().get("password").asText()); + } + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && !adminSettings.getJsonValue().has("privateKey")) { + AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); + if (vcSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("privateKey", vcSettings.getJsonValue().get("privateKey").asText()); + if (!adminSettings.getJsonValue().has("privateKeyPassword") && vcSettings.getJsonValue().has("privateKeyPassword")) { + ((ObjectNode) adminSettings.getJsonValue()).put("privateKeyPassword", vcSettings.getJsonValue().get("privateKeyPassword").asText()); + } + } + } } - return adminSettingsDao.save(tenantId, adminSettings); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 58423645b1..10275a3c1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -25,5 +25,6 @@ import java.util.UUID; */ public interface AdminSettingsRepository extends JpaRepository { - AdminSettingsEntity findByKey(String key); + AdminSettingsEntity findByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 438274e3cc..4a4f7e5440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -46,7 +46,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDaoorg.eclipse.jgit ${jgit.version} + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + ${jgit.version} + From 2eb94fd95e5eca1973a689802af0279124f61a7d Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 12:38:06 +0300 Subject: [PATCH 046/156] Fix beans cycle dependency --- .../DefaultEntitiesVersionControlService.java | 2 -- .../sync/vc/EntitiesVersionControlService.java | 2 ++ .../sync/vc/LocalGitVersionControlService.java | 18 ++++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 1496882031..70bf5b0d5b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -79,8 +79,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntityService entityService; private final TransactionTemplate transactionTemplate; - public static final String SETTINGS_KEY = "entitiesVersionControl"; - @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { var commit = gitService.prepareCommit(user.getTenantId(), request); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 8c18aeb32d..d24fa1411e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -32,6 +32,8 @@ import java.util.List; public interface EntitiesVersionControlService { + String SETTINGS_KEY = "entitiesVersionControl"; + VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index aa51fc3ea1..94976438e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -38,6 +39,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -63,14 +65,14 @@ public class LocalGitVersionControlService implements GitVersionControlService { private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; - private final EntitiesVersionControlService entitiesVersionControlService; + private final AdminSettingsService adminSettingsService; private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = entitiesVersionControlService.getVersionControlSettings(tenantId); + EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); if (settings != null) { try { gitRepositoryService.initRepository(tenantId, settings); @@ -213,6 +215,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } } + private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + } + private List listVersions(TenantId tenantId, String branch, String path) { try { return gitRepositoryService.listVersions(tenantId, branch, path); From 587066cfd32b17e6b0259f147b32987a68c94a63 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 19 May 2022 13:19:43 +0300 Subject: [PATCH 047/156] Ability to remove version control settings --- .../server/controller/AdminController.java | 24 ++++++++++++++++--- .../DefaultEntitiesVersionControlService.java | 7 ++++++ .../vc/EntitiesVersionControlService.java | 2 ++ .../vc/LocalGitVersionControlService.java | 14 +++++++++++ .../dao/settings/AdminSettingsService.java | 2 ++ .../sync/vc/DefaultGitRepositoryService.java | 3 ++- .../service/sync/vc/GitRepositoryService.java | 2 ++ .../sync/vc/GitVersionControlService.java | 2 ++ .../server/dao/settings/AdminSettingsDao.java | 2 ++ .../settings/AdminSettingsServiceImpl.java | 6 +++++ .../sql/settings/AdminSettingsRepository.java | 4 ++++ .../dao/sql/settings/JpaAdminSettingsDao.java | 9 +++++++ 12 files changed, 73 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 237f681819..bf8c656257 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -19,13 +19,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; @@ -38,8 +41,8 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.update.UpdateService; -import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; -import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.DEVICE_ID; @RestController @TbCoreComponent @@ -203,7 +206,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vsSettings") + @PostMapping("/vcSettings") public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); @@ -213,6 +216,21 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", + notes = "Deletes the version control settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vcSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteVersionControlSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); + versionControlService.deleteVersionControlSettings(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 70bf5b0d5b..c69ff94630 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -312,6 +312,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return savedVersionControlSettings; } + @Override + public void deleteVersionControlSettings(TenantId tenantId) { + if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { + gitService.clearRepository(tenantId); + } + } + @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index d24fa1411e..28f095123c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -58,6 +58,8 @@ public interface EntitiesVersionControlService { EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + void deleteVersionControlSettings(TenantId tenantId); + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index 94976438e9..d73157d256 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -111,6 +111,20 @@ public class LocalGitVersionControlService implements GitVersionControlService { } } + @Override + public void clearRepository(TenantId tenantId) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.clearRepository(tenantId); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + @Override public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { var lock = getRepoLock(tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java index d9238c9d7b..77f542ccc3 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java @@ -27,4 +27,6 @@ public interface AdminSettingsService { AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings); + boolean deleteAdminSettings(TenantId tenantId, String key); + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 56ca38ffe4..aac40e0dcb 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -224,7 +224,8 @@ public class DefaultGitRepositoryService implements GitRepositoryService { repositories.put(tenantId, repository); } - private void clearRepository(TenantId tenantId) throws IOException { + @Override + public void clearRepository(TenantId tenantId) throws IOException { GitRepository repository = repositories.get(tenantId); if (repository != null) { FileUtils.deleteDirectory(new File(repository.getDirectory())); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 4b1843e3c7..19754750c2 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -36,6 +36,8 @@ public interface GitRepositoryService { void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void clearRepository(TenantId tenantId) throws IOException; + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index 8345375e4f..f28584c418 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -34,6 +34,8 @@ public interface GitVersionControlService { void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + void clearRepository(TenantId tenantId); + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); void addToCommit(PendingCommit commit, EntityExportData> entityData); diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index 14455cddd4..d211a8340f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -39,4 +39,6 @@ public interface AdminSettingsDao extends Dao { */ AdminSettings findByTenantIdAndKey(UUID tenantId, String key); + boolean removeByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 0bb5564fa6..433d056186 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -62,4 +62,10 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { return adminSettingsDao.save(tenantId, adminSettings); } + @Override + public boolean deleteAdminSettings(TenantId tenantId, String key) { + log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); + Validator.validateString(key, "Incorrect key " + key); + return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 10275a3c1a..0e68327fdf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -27,4 +27,8 @@ public interface AdminSettingsRepository extends JpaRepository Date: Thu, 19 May 2022 13:21:19 +0300 Subject: [PATCH 048/156] Clear repository on settings update --- .../service/sync/vc/DefaultEntitiesVersionControlService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c69ff94630..dcc96b7d17 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -305,6 +305,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new RuntimeException("Failed to load version control settings!", e); } try { + gitService.clearRepository(tenantId); gitService.initRepository(tenantId, savedVersionControlSettings); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); From 10248aa9f64c5d90d2692cf35f7b803b0f11d384 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 19 May 2022 17:49:40 +0300 Subject: [PATCH 049/156] fix of the packge list --- .../server/vc/ThingsboardVersionControlExecutorApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java index 2724a566ee..d30818110d 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -25,7 +25,7 @@ import java.util.Arrays; @SpringBootConfiguration @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.sync.vc"}) +@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) public class ThingsboardVersionControlExecutorApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; From 76af741399fc01f2eb2b735802f00eb2e932f755 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 20 May 2022 15:02:30 +0300 Subject: [PATCH 050/156] UI: Implement git settings form --- .../server/controller/AdminController.java | 8 +- .../DefaultEntitiesVersionControlService.java | 21 +- .../dao/sql/settings/JpaAdminSettingsDao.java | 2 + ui-ngx/src/app/core/http/admin.service.ts | 20 ++ ui-ngx/src/app/core/services/menu.service.ts | 14 +- .../home/pages/admin/admin-routing.module.ts | 14 ++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../version-control-settings.component.html | 110 ++++++++++ .../version-control-settings.component.scss | 33 +++ .../version-control-settings.component.ts | 198 ++++++++++++++++++ ui-ngx/src/app/shared/models/constants.ts | 3 +- .../src/app/shared/models/settings.models.ts | 21 ++ .../assets/locale/locale.constant-en_US.json | 24 ++- 13 files changed, 459 insertions(+), 13 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index bf8c656257..45b8aa5f95 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -207,10 +207,14 @@ public class AdminController extends BaseController { notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vcSettings") - public void saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + public EntitiesVersionControlSettings saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - versionControlService.saveVersionControlSettings(getTenantId(), settings); + EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.saveVersionControlSettings(getTenantId(), settings)); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index dcc96b7d17..d46884ca1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -291,11 +291,21 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + EntitiesVersionControlSettings storedSettings = null; + if (adminSettings != null) { + try { + storedSettings = JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); - AdminSettings adminSettings = new AdminSettings(); - adminSettings.setTenantId(tenantId); - adminSettings.setKey(SETTINGS_KEY); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setTenantId(tenantId); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -341,8 +351,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { if (storedSettings != null) { settings.setPrivateKey(storedSettings.getPrivateKey()); - if (StringUtils.isEmpty(settings.getPrivateKeyPassword()) && - StringUtils.isNotEmpty(storedSettings.getPrivateKeyPassword())) { + if (settings.getPrivateKeyPassword() == null) { settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 7257cfb57b..de4a2fd5ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -51,6 +52,7 @@ public class JpaAdminSettingsDao extends JpaAbstractDao { + return this.http.get(`/api/admin/vcSettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/vcSettings', versionControlSettings, + defaultHttpOptionsFromConfig(config)); + } + + public deleteEntitiesVersionControlSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)); + } + + public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/vcSettings/checkAccess', versionControlSettings, defaultHttpOptionsFromConfig(config)); + } + public checkUpdates(config?: RequestConfig): Observable { return this.http.get(`/api/admin/updates`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 4a52fc66f6..85cf24e04d 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -350,7 +350,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '80px', + height: '120px', icon: 'settings', pages: [ { @@ -366,6 +366,13 @@ export class MenuService { type: 'link', path: '/settings/resources-library', icon: 'folder' + }, + { + id: guid(), + name: 'admin.git-settings', + type: 'link', + path: '/settings/vc', + icon: 'manage_history' } ] } @@ -500,6 +507,11 @@ export class MenuService { name: 'resource.resources-library', icon: 'folder', path: '/settings/resources-library' + }, + { + name: 'admin.git-settings', + icon: 'manage_history', + path: '/settings/vc', } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index ddccbb4da4..6c4963d06e 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -32,6 +32,7 @@ import { ResourcesLibraryTableConfigResolver } from '@home/pages/admin/resource/ import { EntityDetailsPageComponent } from '@home/components/entity/entity-details-page.component'; import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; +import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -183,6 +184,19 @@ const routes: Routes = [ } } ] + }, + { + path: 'vc', + component: VersionControlSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.git-settings', + breadcrumb: { + label: 'admin.git-settings', + icon: 'manage_history' + } + } } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 326b16f950..cb81e56020 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -28,6 +28,7 @@ import { SmsProviderComponent } from '@home/pages/admin/sms-provider.component'; import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dialog.component'; import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; +import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; @NgModule({ declarations: @@ -39,7 +40,8 @@ import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources- SecuritySettingsComponent, OAuth2SettingsComponent, HomeSettingsComponent, - ResourcesLibraryComponent + ResourcesLibraryComponent, + VersionControlSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html new file mode 100644 index 0000000000..b512bb96f5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html @@ -0,0 +1,110 @@ + +
+ + +
+ admin.git-repository-settings + +
+
+ + + +
+ +
+
+ + admin.repository-url + + + admin.repository-url-required + + + + admin.default-branch + + +
+ admin.authentication-settings + + admin.auth-method + + + {{versionControlAuthMethodTranslations.get(method) | translate}} + + + +
+ + common.username + + + + {{ 'admin.change-password-access-token' | translate }} + + + admin.password-access-token + + + +
+
+ + + + {{ 'admin.change-passphrase' | translate }} + + + admin.passphrase + + + +
+
+
+ + + + +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss new file mode 100644 index 0000000000..ede3570e68 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts new file mode 100644 index 0000000000..c2626ec062 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts @@ -0,0 +1,198 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + EntitiesVersionControlSettings, + VersionControlAuthMethod, + versionControlAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; + +@Component({ + selector: 'tb-version-control-settings', + templateUrl: './version-control-settings.component.html', + styleUrls: ['./version-control-settings.component.scss', './settings-card.scss'] +}) +export class VersionControlSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + versionControlSettingsForm: FormGroup; + settings: EntitiesVersionControlSettings = null; + + versionControlAuthMethod = VersionControlAuthMethod; + versionControlAuthMethods = Object.values(VersionControlAuthMethod); + versionControlAuthMethodTranslations = versionControlAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.versionControlSettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: [null, []], + authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.versionControlSettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).subscribe( + (settings) => { + this.settings = settings; + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); + }); + } + + checkAccess(): void { + const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; + this.adminService.checkVersionControlAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-vc-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; + this.adminService.saveEntitiesVersionControlSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-git-settings-title', ), + this.translate.instant('admin.delete-git-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteEntitiesVersionControlSettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.versionControlSettingsForm.reset({ authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + } + ); + } + }); + } + + confirmForm(): FormGroup { + return this.versionControlSettingsForm; + } + + changePasswordChanged() { + if (this.changePassword) { + this.versionControlSettingsForm.get('password').patchValue(''); + this.versionControlSettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.versionControlSettingsForm.get('privateKeyPassword').patchValue(''); + this.versionControlSettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: VersionControlAuthMethod = this.versionControlSettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.versionControlSettingsForm.get('privateKeyFileName').value; + if (authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.versionControlSettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.versionControlSettingsForm.get('password').enable({emitEvent}); + } else { + this.versionControlSettingsForm.get('password').disable({emitEvent}); + } + this.versionControlSettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKey').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.versionControlSettingsForm.get('username').disable({emitEvent}); + this.versionControlSettingsForm.get('password').disable({emitEvent}); + this.versionControlSettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.versionControlSettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.versionControlSettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.versionControlSettingsForm.get('privateKey').clearValidators(); + } else { + this.versionControlSettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.versionControlSettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.versionControlSettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 4bfce6414a..7c970678b8 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -133,7 +133,8 @@ export const HelpLinks = { widgetsConfigAlarm: helpBaseUrl + '/docs/user-guide/ui/dashboards#alarm', widgetsConfigStatic: helpBaseUrl + '/docs/user-guide/ui/dashboards#static', ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', - ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge' + ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', + versionControlSettings: helpBaseUrl + '/docs/user-guide/ui/version-control-settings' } }; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 615d0994c9..8a1b316001 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -396,3 +396,24 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid } return smsProviderConfiguration; } + +export enum VersionControlAuthMethod { + USERNAME_PASSWORD = 'USERNAME_PASSWORD', + PRIVATE_KEY = 'PRIVATE_KEY' +} + +export const versionControlAuthMethodTranslationMap = new Map([ + [VersionControlAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [VersionControlAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +]); + +export interface EntitiesVersionControlSettings { + repositoryUri: string; + defaultBranch: string; + authMethod: VersionControlAuthMethod; + username: string; + password: string; + privateKeyFileName: string; + privateKey: string; + privateKeyPassword: string; +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cf576aedbf..5aaadb2bd6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -311,8 +311,28 @@ "scheme-music-codes": "10 - Music Codes (ISO-2022-JP)", "scheme-extended-kanji-jis": "13 - Extended Kanji JIS (X 0212-1990)", "scheme-korean-graphic-character-set": "14 - Korean Graphic Character Set (KS C 5601/KS X 1001)" - } - }, + }, + "git-settings": "Git settings", + "git-repository-settings": "Git repository settings", + "repository-url": "Repository URL", + "repository-url-required": "Repository URL is required.", + "default-branch": "Default branch name", + "authentication-settings": "Authentication settings", + "auth-method": "Authentication method", + "auth-method-username-password": "Password / access token", + "auth-method-private-key": "Private key", + "password-access-token": "Password / access token", + "change-password-access-token": "Change password / access token", + "private-key": "Private key", + "drop-private-key-file-or": "Drag and drop a private key file or", + "passphrase": "Passphrase", + "enter-passphrase": "Enter passphrase", + "change-passphrase": "Change passphrase", + "check-access": "Check access", + "check-vc-access-success": "Git repository access successfully verified!", + "delete-git-settings-title": "Are you sure you want to delete git settings?", + "delete-git-settings-text": "Be careful, after the confirmation the git settings will be removed and git synchronization feature will be unavailable." + }, "alarm": { "alarm": "Alarm", "alarms": "Alarms", From 7071c7a6bc9deb72bf579e4db868c7d98146c529 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 20 May 2022 17:33:24 +0300 Subject: [PATCH 051/156] Implementation of the Git Queue service client --- .../server/controller/BaseController.java | 21 + .../EntitiesVersionControlController.java | 85 ++-- .../install/SqlDatabaseUpgradeService.java | 12 - .../queue/DefaultTbClusterService.java | 8 + .../service/sync/vc/CommitGitRequest.java | 33 ++ .../DefaultEntitiesVersionControlService.java | 215 +++++---- .../DefaultGitVersionControlQueueService.java | 287 +++++++++++ .../sync/vc/EntitiesContentGitRequest.java | 37 ++ .../vc/EntitiesVersionControlService.java | 19 +- .../sync/vc/EntityContentGitRequest.java | 34 ++ .../vc/GitVersionControlQueueService.java | 57 +++ .../sync/vc/ListBranchesGitRequest.java | 29 ++ .../sync/vc/ListEntitiesGitRequest.java | 30 ++ .../sync/vc/ListVersionsGitRequest.java | 31 ++ .../vc/LocalGitVersionControlService.java | 452 +++++++++--------- .../service/sync/vc/PendingGitRequest.java | 37 ++ .../src/main/resources/thingsboard.yml | 5 +- .../server/cluster/TbClusterService.java | 7 +- common/cluster-api/src/main/proto/queue.proto | 96 +++- .../server/common/msg/queue/ServiceType.java | 2 +- .../queue/discovery/HashPartitionService.java | 10 +- .../queue/kafka/TbKafkaTopicConfigs.java | 6 + .../provider/AwsSqsMonolithQueueFactory.java | 7 + .../provider/AwsSqsTbCoreQueueFactory.java | 7 + .../InMemoryMonolithQueueFactory.java | 6 + .../provider/KafkaMonolithQueueFactory.java | 36 +- .../provider/KafkaTbCoreQueueFactory.java | 20 + .../KafkaTbVersionControlQueueFactory.java | 116 +++++ .../provider/PubSubMonolithQueueFactory.java | 7 + .../provider/PubSubTbCoreQueueFactory.java | 7 + .../RabbitMqMonolithQueueFactory.java | 7 + .../provider/RabbitMqTbCoreQueueFactory.java | 7 + .../ServiceBusMonolithQueueFactory.java | 7 + .../ServiceBusTbCoreQueueFactory.java | 7 + .../queue/provider/TbCoreQueueFactory.java | 8 + .../provider/TbCoreQueueProducerProvider.java | 8 + .../provider/TbQueueProducerProvider.java | 8 + .../TbRuleEngineProducerProvider.java | 6 + .../TbTransportQueueProducerProvider.java | 6 + .../TbVersionControlProducerProvider.java | 84 ++++ .../TbVersionControlQueueFactory.java | 44 ++ .../TbQueueVersionControlSettings.java | 34 ++ .../queue/util/TbVersionControlComponent.java | 26 + common/version-control/pom.xml | 24 - .../sync/vc/GitVersionControlService.java | 11 - .../version-control/src/main/proto/vc.proto | 71 --- 46 files changed, 1579 insertions(+), 498 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java delete mode 100644 common/version-control/src/main/proto/vc.proto diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 195ea9a3ee..709b809823 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -18,6 +18,10 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -27,6 +31,7 @@ import org.springframework.http.MediaType; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -948,4 +953,20 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + protected DeferredResult wrapFuture(ListenableFuture future) { + final DeferredResult deferredResult = new DeferredResult<>(); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(T result) { + deferredResult.setResult(result); + } + + @Override + public void onFailure(Throwable t) { + deferredResult.setErrorResult(t); + } + }, MoreExecutors.directExecutor()); + return deferredResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..4c5c7dad65 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -15,16 +15,26 @@ */ package org.thingsboard.server.controller; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import lombok.Data; import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.device.claim.ClaimResponse; +import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -34,6 +44,7 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.UUID; @@ -90,16 +101,15 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/version") - public VersionCreationResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + public DeferredResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - return versionControlService.saveEntitiesVersion(user, request); + return wrapFuture(versionControlService.saveEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); } } - @ApiOperation(value = "", notes = "" + "```\n[\n" + " {\n" + @@ -108,12 +118,12 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + public DeferredResult> listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId)); } catch (Exception e) { throw handleException(e); } @@ -127,10 +137,10 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + public DeferredResult> listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType)); } catch (Exception e) { throw handleException(e); } @@ -152,9 +162,9 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n```") @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + public DeferredResult> listVersions(@PathVariable String branch) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + return wrapFuture(versionControlService.listVersions(getTenantId(), branch)); } catch (Exception e) { throw handleException(e); } @@ -162,21 +172,21 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping("/entity/{branch}/{entityType}/{versionId}") - public List listEntitiesAtVersion(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable String versionId) throws ThingsboardException { + public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); + return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); } catch (Exception e) { throw handleException(e); } } @GetMapping("/entity/{branch}/{versionId}") - public List listAllEntitiesAtVersion(@PathVariable String branch, - @PathVariable String versionId) throws ThingsboardException { + public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId); + return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); } catch (Exception e) { throw handleException(e); } @@ -216,20 +226,10 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/entity") - public List loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + public DeferredResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { - String versionId = request.getVersionId(); - if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); - } else { - throw new IllegalArgumentException("No versions available in branch"); - } - } - - return versionControlService.loadEntitiesVersion(user, request); + return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); } catch (Exception e) { throw handleException(e); } @@ -252,19 +252,22 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "]\n\n```") @GetMapping("/branches") - public List listBranches() throws ThingsboardException { + public DeferredResult> listBranches() throws ThingsboardException { try { - List remoteBranches = versionControlService.listBranches(getTenantId()); - List infos = new ArrayList<>(); + final TenantId tenantId = getTenantId(); + ListenableFuture> branches = versionControlService.listBranches(tenantId); + return wrapFuture(Futures.transform(branches, remoteBranches -> { + List infos = new ArrayList<>(); - String defaultBranch = versionControlService.getVersionControlSettings(getTenantId()).getDefaultBranch(); - if (StringUtils.isNotEmpty(defaultBranch)) { - remoteBranches.remove(defaultBranch); - infos.add(new BranchInfo(defaultBranch, true)); - } + String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + remoteBranches.remove(defaultBranch); + infos.add(new BranchInfo(defaultBranch, true)); + } - remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); - return infos; + remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + return infos; + }, MoreExecutors.directExecutor())); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index b1bdbf61b8..db2352b706 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -647,18 +647,6 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService log.error("Failed updating schema!!!", e); } break; - case "3.3.4": - try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { - log.info("Updating schema ..."); - schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL); - loadSql(schemaUpdateFile, conn); - log.info("Updating schema settings..."); - conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;"); - log.info("Schema updated."); - } catch (Exception e) { - log.error("Failed updating schema!!!", e); - } - break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2806c35cda..4a52ea2287 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -135,6 +135,14 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToVersionControl(TenantId tenantId, TransportProtos.ToVersionControlServiceMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + toCoreMsgs.incrementAndGet(); + } + @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java new file mode 100644 index 0000000000..b510ffa62c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +public class CommitGitRequest extends PendingGitRequest { + + private final VersionCreateRequest request; + + public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) { + super(tenantId); + this.request = request; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index dcc96b7d17..3fb47a8bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,12 +15,18 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -33,7 +39,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.*; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -56,6 +61,8 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -72,118 +79,137 @@ import java.util.stream.Collectors; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - private final GitVersionControlService gitService; + private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; + private ListeningExecutorService executor; + + @Value("${vc.thread_pool_size:4}") + private int threadPoolSize; + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class)); + } + + @PreDestroy + public void shutdown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + @SuppressWarnings("UnstableApiUsage") @Override - public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var commit = gitService.prepareCommit(user.getTenantId(), request); + public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); - switch (request.getType()) { - case SINGLE_ENTITY: { - SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); - break; - } - case COMPLEX: { - ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; - versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - gitService.deleteAll(commit, entityType); - } + return Futures.transformAsync(pendingCommit, commit -> { + List> gitFutures = new ArrayList<>(); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; + gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + break; + } + case COMPLEX: { + ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; + versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { + gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); + } - if (config.isAllEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - try { - saveEntityData(user, commit, entity.getId(), config); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - } else { - for (UUID entityId : config.getEntityIds()) { - try { - saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) + , 100, entity -> { + try { + gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); + } catch (Exception e) { + throw new RuntimeException(e); + } } } - } - - }); - break; + }); + break; + } } - } - - return gitService.push(commit); + return Futures.transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + }, executor); } - private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { + private ListenableFuture saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - gitService.addToCommit(commit, entityData); + return gitServiceQueue.addToCommit(commit, entityData); } - @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, externalId); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, entityType); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public ListenableFuture> listVersions(TenantId tenantId, String branch) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch); } @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { - return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override - public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return gitService.listEntitiesAtVersion(tenantId, branch, versionId); + public ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId); } + @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) @Override - public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + public ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); - EntityImportResult importResult = transactionTemplate.execute(status -> { - try { - EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), true, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return List.of(VersionLoadResult.builder() - .entityType(importResult.getEntityType()) - .created(importResult.getOldEntity() == null ? 1 : 0) - .updated(importResult.getOldEntity() != null ? 1 : 0) - .deleted(0) - .build()); + ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + Futures.transform(future, entityData -> { + EntityImportResult importResult = transactionTemplate.execute(status -> { + try { + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + }, executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return transactionTemplate.execute(status -> { + return executor.submit(() -> transactionTemplate.execute(status -> { Map results = new HashMap<>(); Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); @@ -199,9 +225,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { int limit = 100; int offset = 0; - List> entityDataList; + List entityDataList; do { - entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) @@ -215,7 +241,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } offset += limit; importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) - .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); } while (entityDataList.size() == limit); } catch (Exception e) { throw new RuntimeException(e); @@ -263,7 +289,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } return new ArrayList<>(results.values()); - }); + })); } default: throw new IllegalArgumentException("Unsupported version load request"); @@ -272,8 +298,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listBranches(TenantId tenantId) throws Exception { - return gitService.listBranches(tenantId); + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + return gitServiceQueue.listBranches(tenantId); } @Override @@ -304,32 +330,35 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } + //TODO: ashvayka +// try { +// gitService.clearRepository(tenantId); +// gitService.initRepository(tenantId, savedVersionControlSettings); +// } catch (Exception e) { +// throw new RuntimeException("Failed to init repository!", e); +// } return savedVersionControlSettings; } @Override public void deleteVersionControlSettings(TenantId tenantId) { - if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { - gitService.clearRepository(tenantId); - } + //TODO: ashvayka +// if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { +// gitService.clearRepository(tenantId); +// } } @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); settings = this.restoreCredentials(settings, storedSettings); - try { - gitService.testRepository(tenantId, settings); - } catch (Exception e) { - throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), - ThingsboardErrorCode.GENERAL); - } + //TODO: ashvayka +// try { +// gitService.testRepository(tenantId, settings); +// } catch (Exception e) { +// throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), +// ThingsboardErrorCode.GENERAL); +// } } private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java new file mode 100644 index 0000000000..410847fb6b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -0,0 +1,287 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; + +@TbCoreComponent +@Service +@RequiredArgsConstructor +public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final TbServiceInfoProvider serviceInfoProvider; + private final TbClusterService clusterService; + private final Map> pendingRequestMap = new HashMap<>(); + + @Override + public ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request) { + SettableFuture future = SettableFuture.create(); + + CommitGitRequest commit = new CommitGitRequest(tenantId, request); + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setPrepareMsg(getCommitPrepareMsg(request)).build() + ).build(), wrap(future, commit)); + + return future; + } + + @Override + public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJson(entityDataJson).build() + ).build() + ).build(), wrap(commit.getFuture(), null)); + return future; + } + + @Override + public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityType, null); + + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setDeleteMsg( + TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() + ).build() + ).build(), wrap(commit.getFuture(), null)); + + return future; + } + + @Override + public ListenableFuture push(CommitGitRequest commit) { + registerAndSend(commit, builder -> builder.setCommitRequest( + CommitRequestMsg.newBuilder().setPushMsg( + TransportProtos.PushMsg.newBuilder().build() + ).build() + ).build(), wrap(commit.getFuture())); + + return commit.getFuture(); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch).setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .build()); + } + + private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { + ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .build()); + } + + private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { + ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture> listBranches(TenantId tenantId) { + ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); + + return request.getFuture(); + } + + private void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); + } else { + throw new RuntimeException("Future is already done!"); + } + } + + @Override + public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { + EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() + , wrap(request.getFuture())); + + return request.getFuture(); +// try { +// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, +// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); +// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } + } + + @Override + public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { + EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); + + registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityType.name()) + .setOffset(offset) + .setLimit(limit) + ).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + private static TbQueueCallback wrap(SettableFuture future) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static TbQueueCallback wrap(SettableFuture future, T value) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + future.set(value); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static String getRelativePath(EntityType entityType, EntityId entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); + } + + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request) { + var tenantId = request.getTenantId(); + var requestId = request.getRequestId(); + return ToVersionControlServiceMsg.newBuilder() + .setNodeId(serviceInfoProvider.getServiceId()) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setRequestIdMSB(requestId.getMostSignificantBits()) + .setRequestIdLSB(requestId.getLeastSignificantBits()); + + } +} + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java new file mode 100644 index 0000000000..2ded932e44 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +import java.util.List; + +@Getter +public class EntitiesContentGitRequest extends PendingGitRequest> { + + private final String versionId; + private final EntityType entityType; + + public EntitiesContentGitRequest(TenantId tenantId, String versionId, EntityType entityType) { + super(tenantId); + this.versionId = versionId; + this.entityType = entityType; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..02afafedad 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; @@ -34,25 +35,23 @@ public interface EntitiesVersionControlService { String SETTINGS_KEY = "entitiesVersionControl"; - VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + ListenableFuture> listVersions(TenantId tenantId, String branch) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; + ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; - List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - - - List listBranches(TenantId tenantId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java new file mode 100644 index 0000000000..c9ce3c76af --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Getter +public class EntityContentGitRequest extends PendingGitRequest { + + private final String versionId; + private final EntityId entityId; + + public EntityContentGitRequest(TenantId tenantId, String versionId, EntityId entityId) { + super(tenantId); + this.versionId = versionId; + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java new file mode 100644 index 0000000000..e7b7171ea0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlQueueService { + + ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request); + + ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); + + ListenableFuture deleteAll(CommitGitRequest pendingCommit, EntityType entityType); + + ListenableFuture push(CommitGitRequest commit); + + ListenableFuture> listVersions(TenantId tenantId, String branch); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + ListenableFuture> listBranches(TenantId tenantId); + + ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); + + ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java new file mode 100644 index 0000000000..c8219641da --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListBranchesGitRequest extends PendingGitRequest> { + + public ListBranchesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java new file mode 100644 index 0000000000..0c7216ef49 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListEntitiesGitRequest extends PendingGitRequest> { + + public ListEntitiesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java new file mode 100644 index 0000000000..0c8e51042b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public class ListVersionsGitRequest extends PendingGitRequest> { + + public ListVersionsGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..66803e062d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -19,18 +19,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +35,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -60,235 +56,235 @@ import java.util.stream.Collectors; @RequiredArgsConstructor @Service @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) -public class LocalGitVersionControlService implements GitVersionControlService { +public class LocalGitVersionControlService { private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); - private final Map pendingCommitMap = new HashMap<>(); - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); - if (settings != null) { - try { - gitRepositoryService.initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - } - - @Override - public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.testRepository(tenantId, settings); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.initRepository(tenantId, settings); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public void clearRepository(TenantId tenantId) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - gitRepositoryService.clearRepository(tenantId); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } finally { - lock.unlock(); - } - } - - @Override - public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { - var lock = getRepoLock(tenantId); - lock.lock(); - try { - var pendingCommit = new PendingCommit(tenantId, request); - PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); - if (old != null) { - gitRepositoryService.abort(old); - } - gitRepositoryService.prepareCommit(pendingCommit); - return pendingCommit; - } finally { - lock.unlock(); - } - } - - @Override - public void deleteAll(PendingCommit commit, EntityType entityType) { - doInsideLock(commit, c -> { - try { - gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - }); - } - - @Override - public void addToCommit(PendingCommit commit, EntityExportData> entityData) { - doInsideLock(commit, c -> { - String entityDataJson; - try { - entityDataJson = jsonWriter.writeValueAsString(entityData); - gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString()), entityDataJson); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - }); - } - - @Override - public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); - } - - @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); - } - - @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); - } - - @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { - try { - return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { - return listEntitiesAtVersion(tenantId, branch, versionId, null); - } - - @Override - public List listBranches(TenantId tenantId) { - return gitRepositoryService.listBranches(tenantId); - } - - @Override - public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { - return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() - .skip(offset).limit(limit) - .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) - .collect(Collectors.toList()); - } - - @Override - public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { - try { - String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, - getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; - } - - private List listVersions(TenantId tenantId, String branch, String path) { - try { - return gitRepositoryService.listVersions(tenantId, branch, path); - } catch (Exception e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } - } - - private void doInsideLock(PendingCommit commit, Consumer r) { - var lock = getRepoLock(commit.getTenantId()); - lock.lock(); - try { - checkCommit(commit); - r.accept(commit); - } finally { - lock.unlock(); - } - } - - private T executeInsideLock(PendingCommit commit, Function c) { - var lock = getRepoLock(commit.getTenantId()); - lock.lock(); - try { - checkCommit(commit); - return c.apply(commit); - } finally { - lock.unlock(); - } - } - - private void checkCommit(PendingCommit commit) { - PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); - if (existing == null || !existing.getTxId().equals(commit.getTxId())) { - throw new ConcurrentModificationException(); - } - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; - } - - private Lock getRepoLock(TenantId tenantId) { - return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); - } +// private final Map pendingCommitMap = new HashMap<>(); +// +// @AfterStartUp +// public void init() { +// DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { +// EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); +// if (settings != null) { +// try { +// gitRepositoryService.initRepository(tenantId, settings); +// } catch (Exception e) { +// log.warn("Failed to init repository for tenant {}", tenantId, e); +// } +// } +// }); +// } +// +// @Override +// public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.testRepository(tenantId, settings); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.initRepository(tenantId, settings); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void clearRepository(TenantId tenantId) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// gitRepositoryService.clearRepository(tenantId); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { +// var lock = getRepoLock(tenantId); +// lock.lock(); +// try { +// var pendingCommit = new PendingCommit(tenantId, request); +// PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); +// if (old != null) { +// gitRepositoryService.abort(old); +// } +// gitRepositoryService.prepareCommit(pendingCommit); +// return pendingCommit; +// } finally { +// lock.unlock(); +// } +// } +// +// @Override +// public void deleteAll(PendingCommit commit, EntityType entityType) { +// doInsideLock(commit, c -> { +// try { +// gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); +// } catch (IOException e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// }); +// } +// +// @Override +// public void addToCommit(PendingCommit commit, EntityExportData> entityData) { +// doInsideLock(commit, c -> { +// String entityDataJson; +// try { +// entityDataJson = jsonWriter.writeValueAsString(entityData); +// gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), +// entityData.getEntity().getId().toString()), entityDataJson); +// } catch (IOException e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// }); +// } +// +// @Override +// public VersionCreationResult push(PendingCommit commit) { +// return executeInsideLock(commit, gitRepositoryService::push); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch) { +// return listVersions(tenantId, branch, (String) null); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch, EntityType entityType) { +// return listVersions(tenantId, branch, getRelativePath(entityType, null)); +// } +// +// @Override +// public List listVersions(TenantId tenantId, String branch, EntityId entityId) { +// return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); +// } +// +// @Override +// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { +// try { +// return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { +// return listEntitiesAtVersion(tenantId, branch, versionId, null); +// } +// +// @Override +// public List listBranches(TenantId tenantId) { +// return gitRepositoryService.listBranches(tenantId); +// } +// +// @Override +// public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { +// return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() +// .skip(offset).limit(limit) +// .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) +// .collect(Collectors.toList()); +// } +// +// @Override +// public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { +// try { +// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, +// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); +// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { +// AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); +// if (adminSettings != null) { +// try { +// return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); +// } catch (Exception e) { +// throw new RuntimeException("Failed to load version control settings!", e); +// } +// } +// return null; +// } +// +// private List listVersions(TenantId tenantId, String branch, String path) { +// try { +// return gitRepositoryService.listVersions(tenantId, branch, path); +// } catch (Exception e) { +// //TODO: analyze and return meaningful exceptions that we can show to the client; +// throw new RuntimeException(e); +// } +// } +// +// private void doInsideLock(PendingCommit commit, Consumer r) { +// var lock = getRepoLock(commit.getTenantId()); +// lock.lock(); +// try { +// checkCommit(commit); +// r.accept(commit); +// } finally { +// lock.unlock(); +// } +// } +// +// private T executeInsideLock(PendingCommit commit, Function c) { +// var lock = getRepoLock(commit.getTenantId()); +// lock.lock(); +// try { +// checkCommit(commit); +// return c.apply(commit); +// } finally { +// lock.unlock(); +// } +// } +// +// private void checkCommit(PendingCommit commit) { +// PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); +// if (existing == null || !existing.getRequestId().equals(commit.getRequestId())) { +// throw new ConcurrentModificationException(); +// } +// } +// +// private String getRelativePath(EntityType entityType, String entityId) { +// String path = entityType.name().toLowerCase(); +// if (entityId != null) { +// path += "/" + entityId + ".json"; +// } +// return path; +// } +// +// private Lock getRepoLock(TenantId tenantId) { +// return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java new file mode 100644 index 0000000000..3650f9e113 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.SettableFuture; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Getter +public class PendingGitRequest { + + private final UUID requestId; + private final TenantId tenantId; + private final SettableFuture future; + + public PendingGitRequest(TenantId tenantId) { + this.requestId = UUID.randomUUID(); + this.tenantId = tenantId; + this.future = SettableFuture.create(); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 650f110dd7..1032c79054 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1021,6 +1021,9 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1115,8 +1118,8 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" vc: + thread_pool_size: "${TB_VC_POOL_SIZE:4}" git: - service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" management: diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 22953f259a..e2486f3a7e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -31,8 +31,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueClusterService; @@ -47,9 +48,11 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 50f114856f..cb1a1ab4db 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -618,8 +618,8 @@ message TbSubscriptionUpdateValueListProto { } message TbSubscriptionUpdateTsValue { - int64 ts = 1; - optional string value = 2; + int64 ts = 1; + optional string value = 2; } /** @@ -676,6 +676,95 @@ message EdgeNotificationMsgProto { PostAttributeMsg postAttributesMsg = 12; } +/** + TB Core to Version Control Service + */ +message CommitRequestMsg { + PrepareMsg prepareMsg = 1; + AddMsg addMsg = 2; + DeleteMsg deleteMsg = 3; + PushMsg pushMsg = 4; + AbortMsg abortMsg = 5; +} + +message CommitResponseMsg { + string commitId = 1; + string name = 2; + int32 added = 3; + int32 modified = 4; + int32 removed = 5; +} + +message PrepareMsg { + string commitMsg = 1; + string branchName = 2; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} + +message ListVersionsRequestMsg { + string branchName = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message ListEntitiesRequestMsg { + string branchName = 1; + string versionId = 2; + string entityType = 3; +} + +message ListBranchesRequestMsg { +} + +message EntityContentRequestMsg { + string versionId = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message EntitiesContentRequestMsg { + string versionId = 1; + string entityType = 2; + int32 offset = 3; + int32 limit = 4; +} + +message ToVersionControlServiceMsg { + string nodeId = 1; + int64 tenantIdMSB = 2; + int64 tenantIdLSB = 3; + int64 requestIdMSB = 4; + int64 requestIdLSB = 5; + CommitRequestMsg commitRequest = 6; + ListVersionsRequestMsg listVersionRequest = 7; + ListEntitiesRequestMsg listEntitiesRequest = 8; + ListBranchesRequestMsg listBranchesRequest = 9; + EntityContentRequestMsg entityContentRequest = 10; + EntitiesContentRequestMsg entitiesContentRequest = 11; +} + +message VersionControlResponseMsg { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + CommitResponseMsg commitResponse = 3; +} + /** * Main messages; */ @@ -730,6 +819,7 @@ message ToCoreNotificationMsg { bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; + VersionControlResponseMsg vcResponseMsg = 7; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -793,3 +883,5 @@ message ToOtaPackageStateServiceMsg { int64 otaPackageIdLSB = 7; string type = 8; } + + diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 1c05ac7f66..7f276e7e3f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR, TB_VC_EXECUTOR; public static ServiceType of(String serviceType) { return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index ccf8b9dd92..e2325be878 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -54,6 +54,10 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; + @Value("${queue.vc.topic}") + private String vcTopic; + @Value("${queue.vc.partitions:10}") + private Integer vcPartitions; @Value("${queue.partitions.hash_function_name:murmur3_128}") private String hashFunctionName; @@ -97,6 +101,10 @@ public class HashPartitionService implements PartitionService { partitionSizesMap.put(coreKey, corePartitions); partitionTopicsMap.put(coreKey, coreTopic); + QueueKey vcKey = new QueueKey(ServiceType.TB_VC_EXECUTOR); + partitionSizesMap.put(vcKey, vcPartitions); + partitionTopicsMap.put(vcKey, vcTopic); + List queueRoutingInfoList; String serviceType = serviceInfoProvider.getServiceType(); @@ -401,7 +409,7 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); - } else if (ServiceType.TB_CORE.equals(serviceType)) { + } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 4c9194b675..75d0bd9b2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -40,6 +40,9 @@ public class TbKafkaTopicConfigs { private String jsExecutorProperties; @Value("${queue.kafka.topic-properties.ota-updates:}") private String fwUpdatesProperties; + @Value("${queue.kafka.topic-properties.version-control:}") + private String vcProperties; + @Getter private Map coreConfigs; @@ -53,6 +56,8 @@ public class TbKafkaTopicConfigs { private Map jsExecutorConfigs; @Getter private Map fwUpdatesConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -62,6 +67,7 @@ public class TbKafkaTopicConfigs { notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); fwUpdatesConfigs = getConfigs(fwUpdatesProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 660173c71d..d02b58aa2b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -205,6 +206,12 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 81736c2b55..89b45e2822 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fe87fc2886..98c1a8029e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -146,6 +146,12 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 104c531e12..b24ae700bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,6 +52,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicLong; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbKafkaSettings kafkaSettings; @@ -68,6 +70,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -76,6 +79,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -85,6 +90,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.notificationsTopicService = notificationsTopicService; @@ -95,6 +101,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -103,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -155,6 +163,19 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("monolith-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { String queueName = configuration.getName(); @@ -311,6 +332,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -331,5 +362,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 58dec0792a..f476a068d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -50,6 +51,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -65,6 +67,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; @@ -74,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; public KafkaTbCoreQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -82,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { @@ -92,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; @@ -101,6 +107,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -282,6 +289,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -302,5 +319,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..598f86d0f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-vc-executor'") +public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbKafkaConsumerStatsService consumerStatsService, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + this.consumerStatsService = consumerStatsService; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("tb-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index aceb9d4f0d..a57e30b655 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -207,6 +208,12 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 18a6668581..6cf9aa9a45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 2a724289bb..d4839110e8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -205,6 +206,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e728be6085..1dca3ba551 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -169,6 +170,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 88bb0a4045..f7db6379c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -204,6 +205,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e1eb41b2b2..5b471b2830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 4debe1e280..aadefa0292 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateSer import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -122,4 +123,11 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueProducer> createTransportApiResponseProducer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + /** + * Used to push messages to instances of TB Version Control Service + * + * @return + */ + TbQueueProducer> createVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index ef0b0c38e9..ed7de35274 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -22,6 +22,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -39,6 +40,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toVersionControl; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -52,6 +54,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); } @Override @@ -83,4 +86,9 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + return toVersionControl; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 19ebb4666e..19046c5a2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -70,4 +71,11 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbUsageStatsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 26f9df069e..c21ae99b8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -83,4 +84,9 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 3132ed684f..f9cc139559 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -73,6 +74,11 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + @Override public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java new file mode 100644 index 0000000000..c8ffc1da0d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-vc-executor'") +public class TbVersionControlProducerProvider implements TbQueueProducerProvider { + + private final TbVersionControlQueueFactory tbQueueProvider; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; + + public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java new file mode 100644 index 0000000000..70b16afc68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages from TB Core Service + * + * @return + */ + TbQueueConsumer> createToVersionControlMsgConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java new file mode 100644 index 0000000000..e7f0ce2259 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +public class TbQueueVersionControlSettings { + + @Value("${queue.vc.topic:tb_version_control}") + private String topic; + + @Value("${queue.vc.usage-stats-topic:tb_usage_stats}") + private String usageStatsTopic; + + @Value("${queue.vc.partitions:10}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java new file mode 100644 index 0000000000..e72fe60b6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-vc-executor'") +public @interface TbVersionControlComponent { +} diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index a8d775bcdc..7b46f7ba72 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -86,21 +86,6 @@ org.eclipse.jgit org.eclipse.jgit - - io.grpc - grpc-netty-shaded - provided - - - io.grpc - grpc-protobuf - provided - - - io.grpc - grpc-stub - provided - org.eclipse.jgit org.eclipse.jgit.ssh.apache @@ -126,15 +111,6 @@ - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - - - - thingsboard-repo-deploy diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..56349cf8a5 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -16,15 +16,12 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; @@ -36,14 +33,6 @@ public interface GitVersionControlService { void clearRepository(TenantId tenantId); - PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); - - void addToCommit(PendingCommit commit, EntityExportData> entityData); - - void deleteAll(PendingCommit pendingCommit, EntityType entityType); - - VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); List listVersions(TenantId tenantId, String branch, EntityType entityType); diff --git a/common/version-control/src/main/proto/vc.proto b/common/version-control/src/main/proto/vc.proto deleted file mode 100644 index e8c7e09384..0000000000 --- a/common/version-control/src/main/proto/vc.proto +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -syntax = "proto3"; - -option java_package = "org.thingsboard.server.gen.vc.v1"; -option java_multiple_files = true; -option java_outer_classname = "EdgeProtos"; - -package vc; - -// Interface exported by the ThingsBoard Core. -service TbGitRpcService { - - rpc commit(stream CommitRequestMsg) returns (CommitResponseMsg) {} - -} - - -/** - * Data Structures; - */ -message CommitRequestMsg { - string txId = 1; - PrepareMsg prepareMsg = 2; - AddMsg addMsg = 3; - DeleteMsg deleteMsg = 4; - PushMsg pushMsg = 5; - AbortMsg abortMsg = 6; -} - -message CommitResponseMsg { - string id = 1; - string name = 2; - int32 added = 3; - int32 modified = 4; - int32 removed = 5; -} - -message PrepareMsg { - string tenantId = 1; - string commitMsg = 2; - string branchName = 3; -} - -message AddMsg { - string relativePath = 1; - string entityDataJson = 2; -} - -message DeleteMsg { - string relativePath = 1; -} - -message PushMsg { -} - -message AbortMsg { -} \ No newline at end of file From 841f9d5ff6287af4f6ba3e39b7bff3d09a76ab70 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 20 May 2022 17:50:35 +0300 Subject: [PATCH 052/156] UI: Single entity Git export --- .../http/entities-version-control.service.ts | 40 ++++ .../home/components/home-components.module.ts | 7 +- .../profile/device-profile.component.html | 6 + .../vc/vc-entity-export-dialog.component.html | 77 +++++++ .../vc/vc-entity-export-dialog.component.ts | 105 +++++++++ .../home/dialogs/home-dialogs.service.ts | 17 ++ .../home/pages/asset/asset.component.html | 6 + .../asset/assets-table-config.resolver.ts | 10 + .../pages/customer/customer.component.html | 6 + .../customers-table-config.resolver.ts | 12 ++ .../dashboard/dashboard-form.component.html | 6 + .../dashboards-table-config.resolver.ts | 12 ++ .../device-profiles-table-config.resolver.ts | 12 ++ .../home/pages/device/device.component.html | 6 + .../device/devices-table-config.resolver.ts | 10 + .../pages/rulechain/rulechain.component.html | 6 + .../rulechains-table-config.resolver.ts | 12 ++ .../vc/branch-autocomplete.component.html | 43 ++++ .../vc/branch-autocomplete.component.ts | 199 ++++++++++++++++++ ui-ngx/src/app/shared/models/vc.models.ts | 55 +++++ ui-ngx/src/app/shared/shared.module.ts | 7 +- .../assets/locale/locale.constant-en_US.json | 12 ++ 22 files changed, 662 insertions(+), 4 deletions(-) create mode 100644 ui-ngx/src/app/core/http/entities-version-control.service.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts create mode 100644 ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html create mode 100644 ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts create mode 100644 ui-ngx/src/app/shared/models/vc.models.ts diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts new file mode 100644 index 0000000000..231581a02f --- /dev/null +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -0,0 +1,40 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable } from 'rxjs'; +import { BranchInfo, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; + +@Injectable({ + providedIn: 'root' +}) +export class EntitiesVersionControlService { + + constructor( + private http: HttpClient + ) { + } + + public listBranches(config?: RequestConfig): Observable> { + return this.http.get>('/api/entities/vc/branches', defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); + } +} diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 4a721647ef..a88f0349eb 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -151,6 +151,7 @@ import { DashboardStateComponent } from '@home/components/dashboard-page/dashboa import { EntityDetailsPageComponent } from '@home/components/entity/entity-details-page.component'; import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; +import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-export-dialog.component'; @NgModule({ declarations: @@ -272,7 +273,8 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings DashboardStateDialogComponent, DashboardImageDialogComponent, EmbedDashboardDialogComponent, - DisplayWidgetTypesPanelComponent + DisplayWidgetTypesPanelComponent, + VcEntityExportDialogComponent ], imports: [ CommonModule, @@ -388,7 +390,8 @@ import { WidgetSettingsComponent } from '@home/components/widget/widget-settings DashboardStateDialogComponent, DashboardImageDialogComponent, EmbedDashboardDialogComponent, - DisplayWidgetTypesPanelComponent + DisplayWidgetTypesPanelComponent, + VcEntityExportDialogComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html index 565eb5b160..7e35dbdaf4 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device-profile.component.html @@ -34,6 +34,12 @@ [fxShow]="!isEdit && !entity?.default"> {{'device-profile.set-default' | translate }} + + + + +
+
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+
+
+
+
+
+ + +
+
+ +
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts new file mode 100644 index 0000000000..9d3affea42 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts @@ -0,0 +1,105 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; +import { ErrorStateMatcher } from '@angular/material/core'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; +import { Router } from '@angular/router'; +import { DialogComponent } from '@app/shared/components/dialog.component'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { TranslateService } from '@ngx-translate/core'; + +export interface VcEntityExportDialogData { + entityId: EntityId; +} + +@Component({ + selector: 'tb-vc-entity-export-dialog', + templateUrl: './vc-entity-export-dialog.component.html', + providers: [{provide: ErrorStateMatcher, useExisting: VcEntityExportDialogComponent}], + styleUrls: [] +}) +export class VcEntityExportDialogComponent extends DialogComponent + implements OnInit, ErrorStateMatcher { + + exportFormGroup: FormGroup; + + submitted = false; + + createResult: VersionCreationResult; + + createResultMessage: SafeHtml; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: VcEntityExportDialogData, + @SkipSelf() private errorStateMatcher: ErrorStateMatcher, + public dialogRef: MatDialogRef, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private domSanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store, router, dialogRef); + + this.exportFormGroup = this.fb.group({ + branch: [null, [Validators.required]], + versionName: [null, [Validators.required]], + saveRelations: [false, []] + }); + } + + ngOnInit(): void { + } + + isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { + const originalErrorState = this.errorStateMatcher.isErrorState(control, form); + const customErrorState = !!(control && control.invalid && this.submitted); + return originalErrorState || customErrorState; + } + + cancel(): void { + this.dialogRef.close(); + } + + export(): void { + this.submitted = true; + const request: SingleEntityVersionCreateRequest = { + entityId: this.data.entityId, + branch: this.exportFormGroup.get('branch').value, + versionName: this.exportFormGroup.get('versionName').value, + config: { + saveRelations: this.exportFormGroup.get('saveRelations').value + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + this.createResult = result; + const message = this.translate.instant('version-control.export-entity-version-result-message', + {name: result.version.name, commitId: result.version.id}); + this.createResultMessage = this.domSanitizer.bypassSecurityTrustHtml(message); + }); + } +} diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 6dc8746b2e..1730c69084 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -22,6 +22,11 @@ import { ImportDialogCsvComponent, ImportDialogCsvData } from '@home/components/import-export/import-dialog-csv.component'; +import { EntityId } from '@shared/models/id/entity-id'; +import { + VcEntityExportDialogComponent, + VcEntityExportDialogData +} from '@home/components/vc/vc-entity-export-dialog.component'; @Injectable() export class HomeDialogsService { @@ -41,6 +46,17 @@ export class HomeDialogsService { } } + public exportVcEntity(entityId: EntityId): Observable { + return this.dialog.open(VcEntityExportDialogComponent, + { + disableClose: true, + panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], + data: { + entityId + } + }).afterClosed(); + } + private openImportDialogCSV(entityType: EntityType, importTitle: string, importFileLabel: string): Observable { return this.dialog.open(ImportDialogCsvComponent, { @@ -53,4 +69,5 @@ export class HomeDialogsService { } }).afterClosed(); } + } diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 00ed882cd9..75171b4532 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -46,6 +46,12 @@ [fxShow]="!isEdit && assetScope === 'edge'"> {{ 'edge.unassign-from-edge' | translate }} + + + + + + + + + + + + {{ 'version-control.branch-required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts new file mode 100644 index 0000000000..421710181a --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -0,0 +1,199 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable, of } from 'rxjs'; +import { + catchError, + debounceTime, + distinctUntilChanged, + map, + publishReplay, + refCount, + switchMap, + tap +} from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { BranchInfo } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; + +@Component({ + selector: 'tb-branch-autocomplete', + templateUrl: './branch-autocomplete.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => BranchAutocompleteComponent), + multi: true + }] +}) +export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit, AfterViewInit { + + branchFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + @Input() + selectDefaultBranch = true; + + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; + + filteredBranches: Observable>; + + branches: Observable>; + + searchText = ''; + + private dirty = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder) { + this.branchFormGroup = this.fb.group({ + branch: [null, []] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + + this.branches = null; + this.filteredBranches = this.branchFormGroup.get('branch').valueChanges + .pipe( + debounceTime(150), + distinctUntilChanged(), + tap(value => { + this.updateView(value); + }), + map(value => value ? value : ''), + switchMap(branch => this.fetchBranches(branch)) + ); + } + + ngAfterViewInit(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } + + selectDefaultBranchIfNeeded(): void { + if (this.selectDefaultBranch && !this.modelValue) { + this.getBranches().subscribe( + (data) => { + if (data && data.length) { + const defaultBranch = data.find(branch => branch.default); + if (defaultBranch) { + this.modelValue = defaultBranch.name; + this.branchFormGroup.get('branch').patchValue(this.modelValue, {emitEvent: false}); + this.propagateChange(this.modelValue); + } + } + } + ); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + if (value != null) { + this.branchFormGroup.get('branch').patchValue(value, {emitEvent: false}); + } else { + this.branchFormGroup.get('branch').patchValue('', {emitEvent: false}); + this.selectDefaultBranchIfNeeded(); + } + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.branchFormGroup.get('branch').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + updateView(value: string | null) { + if (this.modelValue !== value) { + this.modelValue = value; + this.propagateChange(this.modelValue); + } + } + + displayBranchFn(branch?: string): string | undefined { + return branch ? branch : undefined; + } + + fetchBranches(searchText?: string): Observable> { + this.searchText = searchText; + return this.getBranches().pipe( + map(branches => branches.map(branch => branch.name).filter(branchName => { + return searchText ? branchName.toUpperCase().startsWith(searchText.toUpperCase()) : true; + })) + ); + } + + getBranches(): Observable> { + if (!this.branches) { + const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading: true, ignoreErrors: true}); + this.branches = branchesObservable.pipe( + catchError(() => of([] as Array)), + publishReplay(1), + refCount() + ); + } + return this.branches; + } + + clear() { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); + setTimeout(() => { + this.branchInput.nativeElement.blur(); + this.branchInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts new file mode 100644 index 0000000000..0c890ff9b4 --- /dev/null +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -0,0 +1,55 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from '@shared/models/id/entity-id'; + +export interface VersionCreateConfig { + saveRelations: boolean; +} + +export enum VersionCreateRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + COMPLEX = 'COMPLEX' +} + +export interface VersionCreateRequest { + versionName: string; + branch: string; + type: VersionCreateRequestType; +} + +export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { + entityId: EntityId; + config: VersionCreateConfig; + type: VersionCreateRequestType.SINGLE_ENTITY; +} + +export interface BranchInfo { + name: string; + default: boolean; +} + +export interface EntityVersion { + id: string; + name: string; +} + +export interface VersionCreationResult { + version: EntityVersion; + added: number; + modified: number; + removed: number; +} diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 8d8de8e74a..7e73452b18 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -163,6 +163,7 @@ import { HtmlComponent } from '@shared/components/html.component'; import { SafePipe } from '@shared/pipe/safe.pipe'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { MultipleImageInputComponent } from '@shared/components/multiple-image-input.component'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { return markedOptionsService; @@ -284,7 +285,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + BranchAutocompleteComponent ], imports: [ CommonModule, @@ -484,7 +486,8 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) WidgetsBundleSearchComponent, CopyButtonComponent, TogglePasswordComponent, - ProtobufContentComponent + ProtobufContentComponent, + BranchAutocompleteComponent ] }) export class SharedModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5aaadb2bd6..00880644e4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3049,6 +3049,18 @@ "json-value-invalid": "JSON value has an invalid format", "json-value-required": "JSON value is required." }, + "version-control": { + "branch": "Branch", + "select-branch": "Select branch", + "branch-required": "Branch is required", + "export-entity-version": "Export entity version", + "entity-version-exported": "Entity version successfully exported", + "version-name": "Version name", + "version-name-required": "Version name is required", + "export-entity-relations": "Export entity relations", + "export-entity-version-result-message": "Entity exported with version '{{name}}' and commit id '{{commitId}}'.", + "export-to-git": "Export to Git" + }, "widget": { "widget-library": "Widgets Library", "widget-bundle": "Widgets Bundle", From 421624f3d48f9a0793077582d68d3fdc98197657 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 12:41:14 +0300 Subject: [PATCH 053/156] Fixed sql test data --- dao/src/test/resources/sql/system-data.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dao/src/test/resources/sql/system-data.sql b/dao/src/test/resources/sql/system-data.sql index 339d6e94ba..8e38b15010 100644 --- a/dao/src/test/resources/sql/system-data.sql +++ b/dao/src/test/resources/sql/system-data.sql @@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1 '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); /** System settings **/ -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{ "baseUrl": "http://localhost:8080" }' ); -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{ "mailFrom": "Thingsboard ", "smtpProtocol": "smtp", "smtpHost": "localhost", From 4472bd7f7c3d70f7a75a3b26bd0a81769789ce6b Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 23 May 2022 12:50:59 +0300 Subject: [PATCH 054/156] Initial implementation of the ClusterVersionControlService --- .../server/actors/ActorSystemContext.java | 2 +- .../DeviceProfileMsgConstructor.java | 2 +- .../queue/DefaultTbClusterService.java | 3 +- .../queue/DefaultTbCoreConsumerService.java | 13 +- .../DefaultTbRuleEngineConsumerService.java | 3 +- .../processing/AbstractConsumerService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 63 ++--- .../DefaultGitVersionControlQueueService.java | 79 ++++++- .../vc/EntitiesVersionControlService.java | 5 +- .../vc/GitVersionControlQueueService.java | 9 + .../service/sync/vc/PendingGitRequest.java | 1 - .../service/sync/vc/VoidGitRequest.java | 29 +++ .../transport/DefaultTransportApiService.java | 2 +- .../src/main/resources/thingsboard.yml | 2 + common/cluster-api/src/main/proto/queue.proto | 24 +- .../vc/EntitiesVersionControlSettings.java | 7 +- common/queue/pom.xml | 4 + .../InMemoryMonolithQueueFactory.java | 13 +- .../util/DataDecodingEncodingService.java | 2 +- .../queue}/util/ProtoWithFSTService.java | 3 +- .../service/ProtoTransportEntityService.java | 5 +- common/transport/transport-api/pom.xml | 4 - .../DefaultTransportDeviceProfileCache.java | 2 +- .../DefaultTransportResourceCache.java | 2 +- .../service/DefaultTransportService.java | 2 +- .../DefaultTransportTenantProfileCache.java | 2 +- common/version-control/pom.xml | 12 +- .../sync/vc/ClusterVersionControlService.java | 22 ++ .../DefaultClusterVersionControlService.java | 219 ++++++++++++++++++ .../sync/vc/GitVersionControlService.java | 52 ----- .../sync/vc/VersionControlRequestCtx.java | 49 ++++ 31 files changed, 505 insertions(+), 134 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => queue/src/main/java/org/thingsboard/server/queue}/util/DataDecodingEncodingService.java (93%) rename common/{transport/transport-api/src/main/java/org/thingsboard/server/common/transport => queue/src/main/java/org/thingsboard/server/queue}/util/ProtoWithFSTService.java (93%) create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java delete mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java create mode 100644 common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 500ad60524..733cf75e64 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -48,7 +48,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java index 77dc121e38..7943ec0595 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java @@ -20,7 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 4a52ea2287..6b475f5a5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -140,6 +140,7 @@ public class DefaultTbClusterService implements TbClusterService { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); log.trace("PUSHING msg: {} to:{}", msg, tpi); producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + //TODO: ashvayka toCoreMsgs.incrementAndGet(); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index bec0dceed3..4cf7f86097 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; @@ -74,6 +74,8 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.GitVersionControlQueueService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -116,6 +118,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; @@ -137,7 +140,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { - if (!request.getFuture().isDone()) { - pendingRequestMap.putIfAbsent(request.getRequestId(), request); - clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); - } else { - throw new RuntimeException("Future is already done!"); - } - } - @Override public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); @@ -218,6 +214,15 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu // } } + private void registerAndSend(PendingGitRequest request, Function enrichFunction, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request)), callback); + } else { + throw new RuntimeException("Future is already done!"); + } + } + @Override public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); @@ -233,6 +238,56 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return request.getFuture(); } + @Override + public ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture clearRepository(TenantId tenantId) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public void processResponse(VersionControlResponseMsg vcResponseMsg) { + UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); + PendingGitRequest request = pendingRequestMap.get(requestId); + if (request == null) { + log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); + return; + } else { + log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + } + var future = request.getFuture(); + if (!StringUtils.isEmpty(vcResponseMsg.getError())) { + future.setException(new RuntimeException(vcResponseMsg.getError())); + } else { + if (vcResponseMsg.hasGenericResponse()) { + future.set(null); + } + } + } + private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 02afafedad..bb42e44f82 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadReques import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; +import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { @@ -57,8 +58,8 @@ public interface EntitiesVersionControlService { EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); - void deleteVersionControlSettings(TenantId tenantId); + void deleteVersionControlSettings(TenantId tenantId) throws Exception; - void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; + void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index e7b7171ea0..f09b383d87 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -21,10 +21,12 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; import java.util.List; @@ -54,4 +56,11 @@ public interface GitVersionControlQueueService { ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + ListenableFuture clearRepository(TenantId tenantId); + + void processResponse(VersionControlResponseMsg vcResponseMsg); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java index 3650f9e113..09bc272828 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; -import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; import java.util.UUID; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java new file mode 100644 index 0000000000..d829960ab4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; + +import java.util.List; + +public class VoidGitRequest extends PendingGitRequest { + + public VoidGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 6c4adfc5ae..f8b42ecc1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -65,7 +65,7 @@ import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProvisionService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 1032c79054..3fb48d2789 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1024,6 +1024,8 @@ queue: vc: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:2000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index cb1a1ab4db..7a5030277e 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -745,24 +745,34 @@ message EntitiesContentRequestMsg { int32 limit = 4; } +message GenericRepositoryRequestMsg {} + +message GenericRepositoryResponseMsg {} + message ToVersionControlServiceMsg { string nodeId = 1; int64 tenantIdMSB = 2; int64 tenantIdLSB = 3; int64 requestIdMSB = 4; int64 requestIdLSB = 5; - CommitRequestMsg commitRequest = 6; - ListVersionsRequestMsg listVersionRequest = 7; - ListEntitiesRequestMsg listEntitiesRequest = 8; - ListBranchesRequestMsg listBranchesRequest = 9; - EntityContentRequestMsg entityContentRequest = 10; - EntitiesContentRequestMsg entitiesContentRequest = 11; + bytes vcSettings = 6; + GenericRepositoryRequestMsg initRepositoryRequest = 7; + GenericRepositoryRequestMsg testRepositoryRequest = 8; + GenericRepositoryRequestMsg clearRepositoryRequest = 9; + CommitRequestMsg commitRequest = 10; + ListVersionsRequestMsg listVersionRequest = 11; + ListEntitiesRequestMsg listEntitiesRequest = 12; + ListBranchesRequestMsg listBranchesRequest = 13; + EntityContentRequestMsg entityContentRequest = 14; + EntitiesContentRequestMsg entitiesContentRequest = 15; } message VersionControlResponseMsg { int64 requestIdMSB = 1; int64 requestIdLSB = 2; - CommitResponseMsg commitResponse = 3; + string error = 3; + GenericRepositoryResponseMsg genericResponse = 4; + CommitResponseMsg commitResponse = 5; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..bc098fbe1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,8 +16,13 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + +import java.io.Serializable; + @Data -public class EntitiesVersionControlSettings { +public class EntitiesVersionControlSettings implements Serializable { + private static final long serialVersionUID = -3211552851889198721L; + private String repositoryUri; private VersionControlAuthMethod authMethod; private String username; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e2197d566f..a625ec6e93 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,6 +120,10 @@ com.google.protobuf protobuf-java-util + + de.ruedigermoeller + fst + org.apache.curator curator-recipes diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 98c1a8029e..b6b89fcc2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -37,28 +37,32 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueVersionControlSettings vcSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, InMemoryStorage storage) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; + this.vcSettings = vcSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -91,6 +95,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, vcSettings.getTopic()); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new InMemoryTbQueueConsumer<>(storage, configuration.getTopic()); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java index ed0572e8e5..8ebcc39cab 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import java.util.Optional; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index 1c5eec383c..f1c55973db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import java.util.Optional; diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index c75de25267..a10c3e4969 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -24,14 +24,11 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @TbSnmpTransportComponent @Service diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 8b3f9feb22..51373e2a4a 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -64,10 +64,6 @@ com.google.code.gson gson - - de.ruedigermoeller - fst - org.slf4j slf4j-api diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index b79306199f..2b0f73343d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java index 666941b143..b4187432e8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java @@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.TransportResourceCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 5b29d3c3ff..3256039e9a 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -69,7 +69,7 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 4cfefb7a6f..bb7f8f3f4d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 7b46f7ba72..294141db8f 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -36,6 +36,14 @@ + + org.thingsboard.common + data + + + org.thingsboard.common + queue + org.springframework spring-core @@ -105,10 +113,6 @@ awaitility test - - org.thingsboard.common - data - diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java new file mode 100644 index 0000000000..fc134e1850 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.springframework.context.ApplicationListener; + +public interface ClusterVersionControlService extends ApplicationListener { +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java new file mode 100644 index 0000000000..73089acdf5 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -0,0 +1,219 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbVersionControlComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@Slf4j +@TbVersionControlComponent +@Service +@RequiredArgsConstructor +public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final TbVersionControlQueueFactory queueFactory; + private final DataDecodingEncodingService encodingService; + private final GitRepositoryService vcService; + private final NotificationsTopicService notificationsTopicService; + + private volatile ExecutorService consumerExecutor; + private volatile TbQueueConsumer> consumer; + private volatile TbQueueProducer> producer; + private volatile boolean stopped = false; + + @Value("${queue.vc.poll-interval:25}") + private long pollDuration; + @Value("${queue.vc.pack-processing-timeout:60000}") + private long packProcessingTimeout; + + @PostConstruct + public void init() { + consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); + producer = queueFactory.createTbCoreNotificationsMsgProducer(); + consumer = queueFactory.createToVersionControlMsgConsumer(); + } + + @PreDestroy + public void stop() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + //TODO: cleanup repositories that we no longer manage in this node. + consumer.subscribe(event.getPartitions()); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent event) { + consumerExecutor.execute(() -> consumerLoop(consumer)); + } + + void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msgWrapper : msgs) { + ToVersionControlServiceMsg msg = msgWrapper.getValue(); + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(new VersionControlRequestCtx(msg, null)); + } else { + VersionControlRequestCtx ctx = new VersionControlRequestCtx(msg, getEntitiesVersionControlSettings(msg)); + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } + } + } +// ConcurrentMap> pendingMap = msgs.stream().collect( +// Collectors.toConcurrentMap(s -> UUID.randomUUID(), Function.identity())); +// CountDownLatch processingTimeoutLatch = new CountDownLatch(1); +// TbPackProcessingContext> ctx = new TbPackProcessingContext<>( +// processingTimeoutLatch, pendingMap, new ConcurrentHashMap<>()); +// pendingMap.forEach((id, msg) -> { +// log.trace("[{}] Creating downlink callback for message: {}", id, msg.getValue()); +// TbCallback callback = new TbPackCallback<>(id, ctx); +// try { +// handleDownlink(id, msg, callback); +// } catch (Throwable e) { +// log.warn("[{}] Failed to process notification: {}", id, msg, e); +// callback.onFailure(e); +// } +// }); +// if (!processingTimeoutLatch.await(processingTimeout, TimeUnit.MILLISECONDS)) { +// ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process downlink: {}", id, msg.getValue())); +// ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process downlink: {}", id, msg.getValue())); +// } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain version control requests from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new version control messages", e2); + } + } + } + } + log.info("TB Version Control request consumer stopped."); + } + + private void handleClearRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.clearRepository(ctx.getTenantId()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void handleInitRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + + private void handleTestRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.testRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void reply(VersionControlRequestCtx ctx, Optional e) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); + TransportProtos.VersionControlResponseMsg.Builder builder = TransportProtos.VersionControlResponseMsg.newBuilder() + .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) + .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); + if (e.isPresent()) { + builder.setError(e.get().getMessage()); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); + log.trace("PUSHING msg: {} to: {}", msg, tpi); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + } + + private EntitiesVersionControlSettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + if (settingsOpt.isPresent()) { + return settingsOpt.get(); + } else { + log.warn("Failed to parse VC settings: {}", msg.getVcSettings()); + throw new RuntimeException("Failed to parse vc settings!"); + } + } + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java deleted file mode 100644 index 56349cf8a5..0000000000 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; - -import java.util.List; - -public interface GitVersionControlService { - - void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); - - void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); - - void clearRepository(TenantId tenantId); - - List listVersions(TenantId tenantId, String branch); - - List listVersions(TenantId tenantId, String branch, EntityType entityType); - - List listVersions(TenantId tenantId, String branch, EntityId entityId); - - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); - - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); - - List listBranches(TenantId tenantId); - - EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); - - List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); - -} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java new file mode 100644 index 0000000000..6541a2978a --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; + +import java.util.UUID; + +@RequiredArgsConstructor +@Data +public class VersionControlRequestCtx { + private final String nodeId; + private final UUID requestId; + private final TenantId tenantId; + private final EntitiesVersionControlSettings settings; + + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, EntitiesVersionControlSettings settings) { + this.nodeId = msg.getNodeId(); + this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); + this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + this.settings = settings; + } + + @Override + public String toString() { + return "VersionControlRequestCtx{" + + "nodeId='" + nodeId + '\'' + + ", requestId=" + requestId + + ", tenantId=" + tenantId + + '}'; + } +} From 2a89fc67aea28844c65444e3da579963c4a56254 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 13:20:00 +0300 Subject: [PATCH 055/156] Git api usage improvements --- .../DefaultEntitiesVersionControlService.java | 29 ++++++----- .../vc/LocalGitVersionControlService.java | 20 +++++++- .../src/main/resources/thingsboard.yml | 1 + .../vc/EntitiesVersionControlSettings.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 51 ++++++++++--------- .../server/service/sync/vc/GitRepository.java | 45 ++++++++-------- .../service/sync/vc/GitRepositoryService.java | 2 + .../server/service/sync/vc/PendingCommit.java | 2 + 8 files changed, 93 insertions(+), 60 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index d46884ca1e..4aedfb9699 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.*; -import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; - -import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; -import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import java.util.ArrayList; import java.util.HashMap; @@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; - private final EntityService entityService; private final TransactionTemplate transactionTemplate; @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index d73157d256..fa767bd1d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService { if (old != null) { gitRepositoryService.abort(old); } - gitRepositoryService.prepareCommit(pendingCommit); + try { + gitRepositoryService.prepareCommit(pendingCommit); + } catch (Exception e) { + pendingCommitMap.remove(tenantId); + gitRepositoryService.cleanUp(pendingCommit); + throw e; + } return pendingCommit; } finally { lock.unlock(); @@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { @Override public VersionCreationResult push(PendingCommit commit) { - return executeInsideLock(commit, gitRepositoryService::push); + VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push); + pendingCommitMap.remove(commit.getTenantId()); + return result; } @Override @@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); r.accept(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } @@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { checkCommit(commit); return c.apply(commit); + } catch (Exception e) { + pendingCommitMap.remove(commit.getTenantId()); + gitRepositoryService.cleanUp(commit); + throw e; } finally { lock.unlock(); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 654929f500..115b4d2ebd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1119,6 +1119,7 @@ vc: git: service: "${JS_VC_GIT_SERVICE:local}" # local/remote repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: endpoints: diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index 4fdff1f413..308f412228 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; + @Data public class EntitiesVersionControlSettings { private String repositoryUri; @@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings { private String privateKey; private String privateKeyPassword; private String defaultBranch; -} \ No newline at end of file +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index aac40e0dcb..e85dee8e2b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; @@ -55,7 +53,7 @@ import java.util.stream.Collectors; @Service public class DefaultGitRepositoryService implements GitRepositoryService { - @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + @Value("${vc.git.repositories-folder}") private String repositoriesFolder; @Value("${vc.git.repos-poll-interval:60}") @@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService { String branch = commit.getRequest().getBranch(); try { repository.fetch(); - if (repository.listBranches().contains(branch)) { - repository.checkout("origin/" + branch, false); - try { - repository.checkout(branch, true); - } catch (RefAlreadyExistsException e) { - repository.checkout(branch, false); - } + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(branch)) { repository.merge(branch); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } } } catch (IOException | GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; @@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setRemoved(status.getRemoved().size()); GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); - repository.push(); + repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch()); result.setVersion(toVersion(gitCommit)); return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); + } + } + + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); } @Override public void abort(PendingCommit commit) { - //TODO: implement; + cleanUp(commit); } @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { - GitRepository repository = checkRepository(tenantId); - return repository.getFileContentAtCommit(relativePath, versionId); + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); } @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); try { - return repository.listBranches(); + return repository.listRemoteBranches(); } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(gitAPIException); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index fa4891d35a..5f1a13ec53 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.sshd.JGitKeyCache; @@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -116,15 +124,18 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch, boolean createBranch) throws GitAPIException { - execute(git.checkout() - .setCreateBranch(createBranch) - .setName(branch)); + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); } - public void reset() throws GitAPIException { + public void resetAndClean() throws GitAPIException { execute(git.reset() .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); } public void merge(String branch) throws IOException, GitAPIException { @@ -137,9 +148,9 @@ public class GitRepository { } - public List listBranches() throws GitAPIException { + public List listRemoteBranches() throws GitAPIException { return execute(git.branchList() - .setListMode(ListBranchCommand.ListMode.ALL)).stream() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .filter(ref -> !ref.getName().equals(Constants.HEAD)) .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(name -> StringUtils.removeStart(name, "origin/")) @@ -209,16 +220,9 @@ public class GitRepository { .setOrphan(true) .setForced(true) .setName(name)); -// Set uncommittedChanges = git.status().call().getUncommittedChanges(); -// if (!uncommittedChanges.isEmpty()) { -// RmCommand rm = git.rm(); -// uncommittedChanges.forEach(rm::addFilepattern); -// execute(rm); -// } -// execute(git.clean()); } - public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] + public void add(String filesPattern) throws GitAPIException { execute(git.add().setUpdate(true).addFilepattern(filesPattern)); execute(git.add().addFilepattern(filesPattern)); } @@ -230,13 +234,14 @@ public class GitRepository { public Commit commit(String message) throws GitAPIException { RevCommit revCommit = execute(git.commit() - .setMessage(message)); // TODO [viacheslav]: set configurable author for commit + .setMessage(message)); return toCommit(revCommit); } - public void push() throws GitAPIException { - execute(git.push()); + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 19754750c2..bbdf976804 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -44,6 +44,8 @@ public interface GitRepositoryService { VersionCreationResult push(PendingCommit commit); + void cleanUp(PendingCommit commit); + void abort(PendingCommit commit); List listBranches(TenantId tenantId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index a30f7514a2..16f3ce5e40 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -27,10 +27,12 @@ public class PendingCommit { private final UUID txId; private final TenantId tenantId; private final VersionCreateRequest request; + private final String workingBranch; public PendingCommit(TenantId tenantId, VersionCreateRequest request) { this.txId = UUID.randomUUID(); this.tenantId = tenantId; this.request = request; + this.workingBranch = txId.toString(); } } From 55966e033e75e7e7d5d13f4d55452f4698f76189 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 23 May 2022 14:04:12 +0300 Subject: [PATCH 056/156] Fixes for vc api --- .../EntitiesVersionControlController.java | 2 +- .../sync/vc/LocalGitVersionControlService.java | 14 ++++++-------- .../common/data/sync/ie/EntityExportData.java | 3 ++- .../common/data/sync/ie/EntityImportResult.java | 6 ------ 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..39a8dc464f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -223,7 +223,7 @@ public class EntitiesVersionControlController extends BaseController { if (versionId == null) { List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); if (versions.size() > 0) { - versionId = versions.get(0).getId(); + request.setVersionId(versions.get(0).getId()); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..8aa4b793df 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -16,21 +16,17 @@ package org.thingsboard.server.service.sync.vc; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -38,7 +34,6 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.queue.util.AfterStartUp; @@ -62,13 +57,16 @@ import java.util.stream.Collectors; @ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) public class LocalGitVersionControlService implements GitVersionControlService { - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); private final GitRepositoryService gitRepositoryService; private final TenantDao tenantDao; private final AdminSettingsService adminSettingsService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); private final Map pendingCommitMap = new HashMap<>(); + private final ObjectMapper jsonMapper = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); + @AfterStartUp public void init() { DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { @@ -165,7 +163,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { doInsideLock(commit, c -> { String entityDataJson; try { - entityDataJson = jsonWriter.writeValueAsString(entityData); + entityDataJson = jsonMapper.writeValueAsString(entityData); gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), entityData.getEntity().getId().toString()), entityDataJson); } catch (IOException e) { @@ -230,7 +228,7 @@ public class LocalGitVersionControlService implements GitVersionControlService { try { String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); - return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + return jsonMapper.readValue(entityDataJson, EntityExportData.class); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index b361a302fc..35fceafe43 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -30,7 +31,7 @@ import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; @JsonIgnoreProperties(ignoreUnknown = true) -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", defaultImpl = EntityExportData.class) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @JsonSubTypes({ @Type(name = "DEVICE", value = DeviceExportData.class), @Type(name = "RULE_CHAIN", value = RuleChainExportData.class) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index ea69425a2c..8d6d9ff7b6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -15,26 +15,20 @@ */ package org.thingsboard.server.common.data.sync.ie; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.sync.JsonTbEntity; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { - @JsonTbEntity private E savedEntity; - @JsonTbEntity private E oldEntity; private EntityType entityType; - @JsonIgnore private ThrowingRunnable saveReferencesCallback = () -> {}; - @JsonIgnore private ThrowingRunnable sendEventsCallback = () -> {}; public void addSaveReferencesCallback(ThrowingRunnable callback) { From 2dcdb4af8e561fedd600b8af5cd86544251e91c1 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 15:36:57 +0300 Subject: [PATCH 057/156] Fix Entities Version Control controller - add TbCoreComponent annotation --- .../server/controller/EntitiesVersionControlController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bba5d20b06..a2b0bf3859 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -41,6 +42,7 @@ import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; @RestController +@TbCoreComponent @RequestMapping("/api/entities/vc") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequiredArgsConstructor From 0d57f2f6c2baf8200066a6c34b67d2bdb87a97ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 23 May 2022 18:19:07 +0300 Subject: [PATCH 058/156] Use page link to get versions list --- .../server/controller/AdminController.java | 14 ++++++ .../EntitiesVersionControlController.java | 50 +++++++++++++------ .../DefaultEntitiesVersionControlService.java | 26 +++++----- .../vc/EntitiesVersionControlService.java | 8 +-- .../vc/LocalGitVersionControlService.java | 18 ++++--- common/version-control/pom.xml | 1 - .../sync/vc/DefaultGitRepositoryService.java | 10 ++-- .../server/service/sync/vc/GitRepository.java | 44 ++++++++++------ .../service/sync/vc/GitRepositoryService.java | 4 +- .../sync/vc/GitVersionControlService.java | 8 +-- 10 files changed, 118 insertions(+), 65 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 45b8aa5f95..ae9bf18bcf 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -203,6 +203,20 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vcSettings/exists") + @ResponseBody + public Boolean versionControlSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index a2b0bf3859..4086788fea 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; import lombok.Data; import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; @@ -25,6 +26,8 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @@ -39,7 +42,7 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -109,13 +112,18 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profile 1 version 1.0\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}/{externalEntityUuid}") - public List listEntityVersions(@PathVariable String branch, - @PathVariable EntityType entityType, - @PathVariable UUID externalEntityUuid) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) + public PageData listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink); } catch (Exception e) { throw handleException(e); } @@ -128,11 +136,16 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Device profiles from dev\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}/{entityType}") - public List listEntityTypeVersions(@PathVariable String branch, - @PathVariable EntityType entityType) throws ThingsboardException { + @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) + public PageData listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink); } catch (Exception e) { throw handleException(e); } @@ -153,10 +166,15 @@ public class EntitiesVersionControlController extends BaseController { " \"name\": \"Devices added\"\n" + " }\n" + "]\n```") - @GetMapping("/version/{branch}") - public List listVersions(@PathVariable String branch) throws ThingsboardException { + @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) + public PageData listVersions(@PathVariable String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page) throws ThingsboardException { try { - return versionControlService.listVersions(getTenantId(), branch); + PageLink pageLink = new PageLink(pageSize, page); + return versionControlService.listVersions(getTenantId(), branch, pageLink); } catch (Exception e) { throw handleException(e); } @@ -223,9 +241,9 @@ public class EntitiesVersionControlController extends BaseController { try { String versionId = request.getVersionId(); if (versionId == null) { - List versions = versionControlService.listVersions(user.getTenantId(), request.getBranch()); - if (versions.size() > 0) { - versionId = versions.get(0).getId(); + PageData versions = versionControlService.listVersions(user.getTenantId(), request.getBranch(), new PageLink(1)); + if (versions.getData().size() > 0) { + versionId = versions.getData().get(0).getId(); } else { throw new IllegalArgumentException("No versions available in branch"); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 4aedfb9699..8f0d11068e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; @@ -134,18 +136,18 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override - public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return gitService.listVersions(tenantId, branch, externalId); + public PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, externalId, pageLink); } @Override - public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return gitService.listVersions(tenantId, branch, entityType); + public PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, entityType, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch) throws Exception { - return gitService.listVersions(tenantId, branch); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitService.listVersions(tenantId, branch, pageLink); } @Override @@ -307,6 +309,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont adminSettings.setKey(SETTINGS_KEY); adminSettings.setTenantId(tenantId); } + try { + gitService.clearRepository(tenantId); + gitService.initRepository(tenantId, versionControlSettings); + } catch (Exception e) { + throw new RuntimeException("Failed to init repository!", e); + } adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); EntitiesVersionControlSettings savedVersionControlSettings; @@ -315,12 +323,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new RuntimeException("Failed to load version control settings!", e); } - try { - gitService.clearRepository(tenantId); - gitService.initRepository(tenantId, savedVersionControlSettings); - } catch (Exception e) { - throw new RuntimeException("Failed to init repository!", e); - } return savedVersionControlSettings; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 28f095123c..a70b78aaf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -37,11 +39,11 @@ public interface EntitiesVersionControlService { VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; - List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + PageData listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; - List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + PageData listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; - List listVersions(TenantId tenantId, String branch) throws Exception; + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java index fa767bd1d5..42a1308901 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -31,6 +31,8 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -183,18 +185,18 @@ public class LocalGitVersionControlService implements GitVersionControlService { } @Override - public List listVersions(TenantId tenantId, String branch) { - return listVersions(tenantId, branch, (String) null); + public PageData listVersions(TenantId tenantId, String branch, PageLink pageLink) { + return listVersions(tenantId, branch, (String) null, pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityType entityType) { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + public PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityType, null), pageLink); } @Override - public List listVersions(TenantId tenantId, String branch, EntityId entityId) { - return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + public PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString()), pageLink); } @Override @@ -249,9 +251,9 @@ public class LocalGitVersionControlService implements GitVersionControlService { return null; } - private List listVersions(TenantId tenantId, String branch, String path) { + private PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) { try { - return gitRepositoryService.listVersions(tenantId, branch, path); + return gitRepositoryService.listVersions(tenantId, branch, path, pageLink); } catch (Exception e) { //TODO: analyze and return meaningful exceptions that we can show to the client; throw new RuntimeException(e); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index b01b466cb5..89c36b1c96 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -60,7 +60,6 @@ com.google.guava guava - provided com.fasterxml.jackson.core diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index e85dee8e2b..b822ff3b5b 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -178,7 +180,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() + return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() .filter(version -> version.getId().equals(versionId)) .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); } @@ -189,11 +191,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); } @Override diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5f1a13ec53..22be22d991 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -15,19 +15,13 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.sshd.common.util.security.SecurityUtils; -import org.eclipse.jgit.api.CloneCommand; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.GitCommand; -import org.eclipse.jgit.api.ListBranchCommand; -import org.eclipse.jgit.api.LogCommand; -import org.eclipse.jgit.api.LsRemoteCommand; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.*; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; @@ -45,6 +39,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -59,6 +55,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class GitRepository { @@ -157,27 +154,25 @@ public class GitRepository { .distinct().collect(Collectors.toList()); } - public List listCommits(String branch, int limit) throws IOException, GitAPIException { - return listCommits(branch, null, limit); + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); } - public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } LogCommand command = git.log() - .add(branchId).setMaxCount(limit) + .add(branchId) .setRevFilter(RevFilter.NO_MERGES); if (StringUtils.isNotEmpty(path)) { command.addPath(path); } - return Streams.stream(execute(command)) - .map(this::toCommit) - .collect(Collectors.toList()); + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink); } - public List listFilesAtCommit(String commitId) throws IOException { return listFilesAtCommit(commitId, null); } @@ -295,6 +290,23 @@ public class GitRepository { return command.call(); } + private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { if (credentialsProvider != null) { transportCommand.setCredentialsProvider(credentialsProvider); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index bbdf976804..8fb0e0f84d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -16,6 +16,8 @@ package org.thingsboard.server.service.sync.vc; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -28,7 +30,7 @@ public interface GitRepositoryService { void prepareCommit(PendingCommit pendingCommit); - List listVersions(TenantId tenantId, String branch, String path) throws Exception; + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java index f28584c418..c21efbb774 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -44,11 +46,11 @@ public interface GitVersionControlService { VersionCreationResult push(PendingCommit commit); - List listVersions(TenantId tenantId, String branch); + PageData listVersions(TenantId tenantId, String branch, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityType entityType); + PageData listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); - List listVersions(TenantId tenantId, String branch, EntityId entityId); + PageData listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); From 1065fd9fbebec782ed7fddc268f26d712cd10917 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 May 2022 11:38:03 +0300 Subject: [PATCH 059/156] Implementation of the API --- .../DefaultGitVersionControlQueueService.java | 49 ++- .../vc/LocalGitVersionControlService.java | 290 ------------------ .../service/sync/vc/PendingGitRequest.java | 4 +- common/cluster-api/src/main/proto/queue.proto | 37 +++ .../data/sync/vc/VersionedEntityInfo.java | 6 + .../DefaultClusterVersionControlService.java | 109 ++++++- .../sync/vc/DefaultGitRepositoryService.java | 8 +- .../service/sync/vc/GitRepositoryService.java | 4 +- 8 files changed, 188 insertions(+), 319 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 7ffcbb9c14..0d3f2a4431 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -21,17 +21,16 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; -import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; @@ -61,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.function.Function; +import java.util.stream.Collectors; @TbCoreComponent @Service @@ -209,6 +209,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override + @SuppressWarnings("rawtypes") public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() @@ -218,14 +219,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() , wrap(request.getFuture())); return request.getFuture(); -// try { -// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, -// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); -// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } } private void registerAndSend(PendingGitRequest request, @@ -237,13 +230,16 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu Function enrichFunction, EntitiesVersionControlSettings settings, TbQueueCallback callback) { if (!request.getFuture().isDone()) { pendingRequestMap.putIfAbsent(request.getRequestId(), request); - clusterService.pushMsgToVersionControl(request.getTenantId(), enrichFunction.apply(newRequestProto(request, settings)), callback); + var requestBody = enrichFunction.apply(newRequestProto(request, settings)); + log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); + clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); } else { throw new RuntimeException("Future is already done!"); } } @Override + @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); @@ -313,10 +309,41 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); ((CommitGitRequest) request).getFuture().set(commitResult); + } else if (vcResponseMsg.hasListBranchesResponse()) { + var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); + } else if (vcResponseMsg.hasListEntitiesResponse()) { + var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); + ((ListEntitiesGitRequest) request).getFuture().set( + listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList())); + } else if (vcResponseMsg.hasListVersionsResponse()) { + var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); + ((ListVersionsGitRequest) request).getFuture().set( + listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList())); + } else if (vcResponseMsg.hasEntityContentResponse()) { + var data = vcResponseMsg.getEntityContentResponse().getData(); + ((EntityContentGitRequest) request).getFuture().set(toData(data)); + } else if (vcResponseMsg.hasEntitiesContentResponse()) { + var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); + ((EntitiesContentGitRequest) request).getFuture() + .set(dataList.stream().map(this::toData).collect(Collectors.toList())); } } } + private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { + return new EntityVersion(proto.getId(), proto.getName()); + } + + private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { + return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); + } + + @SuppressWarnings("rawtypes") + private EntityExportData toData(String data) { + return JacksonUtil.fromString(data, EntityExportData.class); + } + private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java deleted file mode 100644 index d995472531..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java +++ /dev/null @@ -1,290 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; -import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; - -import java.io.IOException; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; - -@Slf4j -@RequiredArgsConstructor -@Service -@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) -public class LocalGitVersionControlService { - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private final GitRepositoryService gitRepositoryService; - private final TenantDao tenantDao; - private final AdminSettingsService adminSettingsService; - private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); - private final Map pendingCommitMap = new HashMap<>(); -// -// @AfterStartUp -// public void init() { -// DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { -// EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); -// if (settings != null) { -// try { -// gitRepositoryService.initRepository(tenantId, settings); -// } catch (Exception e) { -// log.warn("Failed to init repository for tenant {}", tenantId, e); -// } -// } -// }); -// } -// -// @Override -// public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.testRepository(tenantId, settings); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.initRepository(tenantId, settings); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void clearRepository(TenantId tenantId) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// gitRepositoryService.clearRepository(tenantId); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { -// var lock = getRepoLock(tenantId); -// lock.lock(); -// try { -// var pendingCommit = new PendingCommit(tenantId, request); -// PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); -// if (old != null) { -// gitRepositoryService.abort(old); -// } -// gitRepositoryService.prepareCommit(pendingCommit); -// return pendingCommit; -// } finally { -// lock.unlock(); -// } -// } -// -// @Override -// public void deleteAll(PendingCommit commit, EntityType entityType) { -// doInsideLock(commit, c -> { -// try { -// gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); -// } catch (IOException e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// }); -// } -// -// @Override -// public void addToCommit(PendingCommit commit, EntityExportData> entityData) { -// doInsideLock(commit, c -> { -// String entityDataJson; -// try { -// entityDataJson = jsonWriter.writeValueAsString(entityData); -// gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), -// entityData.getEntity().getId().toString()), entityDataJson); -// } catch (IOException e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// }); -// } -// -// @Override -// public VersionCreationResult push(PendingCommit commit) { -// return executeInsideLock(commit, gitRepositoryService::push); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch) { -// return listVersions(tenantId, branch, (String) null); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch, EntityType entityType) { -// return listVersions(tenantId, branch, getRelativePath(entityType, null)); -// } -// -// @Override -// public List listVersions(TenantId tenantId, String branch, EntityId entityId) { -// return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); -// } -// -// @Override -// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { -// try { -// return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// @Override -// public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { -// return listEntitiesAtVersion(tenantId, branch, versionId, null); -// } -// -// @Override -// public List listBranches(TenantId tenantId) { -// return gitRepositoryService.listBranches(tenantId); -// } -// -// @Override -// public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { -// return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() -// .skip(offset).limit(limit) -// .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) -// .collect(Collectors.toList()); -// } -// -// @Override -// public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { -// try { -// String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, -// getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); -// return JacksonUtil.fromString(entityDataJson, EntityExportData.class); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// private EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { -// AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, EntitiesVersionControlService.SETTINGS_KEY); -// if (adminSettings != null) { -// try { -// return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); -// } catch (Exception e) { -// throw new RuntimeException("Failed to load version control settings!", e); -// } -// } -// return null; -// } -// -// private List listVersions(TenantId tenantId, String branch, String path) { -// try { -// return gitRepositoryService.listVersions(tenantId, branch, path); -// } catch (Exception e) { -// //TODO: analyze and return meaningful exceptions that we can show to the client; -// throw new RuntimeException(e); -// } -// } -// -// private void doInsideLock(PendingCommit commit, Consumer r) { -// var lock = getRepoLock(commit.getTenantId()); -// lock.lock(); -// try { -// checkCommit(commit); -// r.accept(commit); -// } finally { -// lock.unlock(); -// } -// } -// -// private T executeInsideLock(PendingCommit commit, Function c) { -// var lock = getRepoLock(commit.getTenantId()); -// lock.lock(); -// try { -// checkCommit(commit); -// return c.apply(commit); -// } finally { -// lock.unlock(); -// } -// } -// -// private void checkCommit(PendingCommit commit) { -// PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); -// if (existing == null || !existing.getRequestId().equals(commit.getRequestId())) { -// throw new ConcurrentModificationException(); -// } -// } -// -// private String getRelativePath(EntityType entityType, String entityId) { -// String path = entityType.name().toLowerCase(); -// if (entityId != null) { -// path += "/" + entityId + ".json"; -// } -// return path; -// } -// -// private Lock getRepoLock(TenantId tenantId) { -// return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); -// } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java index 9b612cee5b..e63ba04f45 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java @@ -24,17 +24,19 @@ import java.util.UUID; @Getter public class PendingGitRequest { + private final long createdTime; private final UUID requestId; private final TenantId tenantId; private final SettableFuture future; public PendingGitRequest(TenantId tenantId) { + this.createdTime = System.currentTimeMillis(); this.requestId = UUID.randomUUID(); this.tenantId = tenantId; this.future = SettableFuture.create(); } - public boolean requiresSettings(){ + public boolean requiresSettings() { return true; } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 6d45698451..54e46eebcc 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -723,15 +723,39 @@ message ListVersionsRequestMsg { int64 entityIdLSB = 4; } +message EntityVersionProto { + int64 ts = 1; + string id = 2; + string name = 3; +} + +message ListVersionsResponseMsg { + repeated EntityVersionProto versions = 1; +} + message ListEntitiesRequestMsg { string branchName = 1; string versionId = 2; string entityType = 3; } +message VersionedEntityInfoProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; +} + +message ListEntitiesResponseMsg { + repeated VersionedEntityInfoProto entities = 1; +} + message ListBranchesRequestMsg { } +message ListBranchesResponseMsg { + repeated string branches = 1; +} + message EntityContentRequestMsg { string versionId = 1; string entityType = 2; @@ -739,6 +763,10 @@ message EntityContentRequestMsg { int64 entityIdLSB = 4; } +message EntityContentResponseMsg { + string data = 1; +} + message EntitiesContentRequestMsg { string versionId = 1; string entityType = 2; @@ -746,6 +774,10 @@ message EntitiesContentRequestMsg { int32 limit = 4; } +message EntitiesContentResponseMsg { + repeated string data = 1; +} + message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -774,6 +806,11 @@ message VersionControlResponseMsg { string error = 3; GenericRepositoryResponseMsg genericResponse = 4; CommitResponseMsg commitResponse = 5; + ListBranchesResponseMsg listBranchesResponse = 6; + ListEntitiesResponseMsg listEntitiesResponse = 7; + ListVersionsResponseMsg listVersionsResponse = 8; + EntityContentResponseMsg entityContentResponse = 9; + EntitiesContentResponseMsg entitiesContentResponse = 10; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 163fe4c6d2..fd278cde1f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -16,10 +16,16 @@ package org.thingsboard.server.common.data.sync.vc; import lombok.Data; +import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.id.EntityId; @Data +@NoArgsConstructor public class VersionedEntityInfo { private EntityId externalId; // etc.. + + public VersionedEntityInfo(EntityId externalId) { + this.externalId = externalId; + } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index b9c595beac..b282799e24 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -23,9 +23,12 @@ import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -58,6 +61,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; +import java.util.stream.Collectors; @Slf4j @TbVersionControlComponent @@ -144,9 +148,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe var newSettings = ctx.getSettings(); if (!newSettings.equals(currentSettings)) { vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } else { + vcService.fetch(ctx.getTenantId()); } if (msg.hasCommitRequest()) { handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); } } } @@ -172,12 +188,71 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe log.info("TB Version Control request consumer stopped."); } - private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg commitRequest) throws Exception { + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { + var entityType = EntityType.valueOf(request.getEntityType()); + String path = getRelativePath(entityType, null); + var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) + .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); + var response = EntitiesContentResponseMsg.newBuilder(); + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + response.addData(data); + } + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + } + + private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { + String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + } + + private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { + String path; + if (StringUtils.isNotEmpty(request.getEntityType())) { + var entityType = EntityType.valueOf(request.getEntityType()); + if (request.getEntityIdLSB() != 0 || request.getEntityIdMSB() != 0) { + path = getRelativePath(entityType, new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + } else { + path = getRelativePath(entityType, null); + } + } else { + path = null; + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() + .addAllVersions(data.stream().map( + v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() + ).collect(Collectors.toList())))); + } + + private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception { + EntityType entityType = StringUtils.isNotEmpty(request.getEntityType()) ? EntityType.valueOf(request.getEntityType()) : null; + var path = entityType != null ? getRelativePath(entityType, null) : null; + var data = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListEntitiesResponse(ListEntitiesResponseMsg.newBuilder() + .addAllEntities(data.stream().map(VersionedEntityInfo::getExternalId).map( + id -> VersionedEntityInfoProto.newBuilder() + .setEntityType(id.getEntityType().name()) + .setEntityIdMSB(id.getId().getMostSignificantBits()) + .setEntityIdLSB(id.getId().getLeastSignificantBits()).build() + ).collect(Collectors.toList())))); + } + + private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { + var branches = vcService.listBranches(ctx.getTenantId()); + reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); + } + + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); - UUID txId = UUID.fromString(commitRequest.getTxId()); - if (commitRequest.hasPrepareMsg()) { - prepareCommit(ctx, txId, commitRequest.getPrepareMsg()); - } else if (commitRequest.hasAbortMsg()) { + UUID txId = UUID.fromString(request.getTxId()); + if (request.hasPrepareMsg()) { + prepareCommit(ctx, txId, request.getPrepareMsg()); + } else if (request.hasAbortMsg()) { PendingCommit current = pendingCommitMap.get(tenantId); if (current != null && current.getTxId().equals(txId)) { doAbortCurrentCommit(tenantId, current); @@ -186,11 +261,11 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe PendingCommit current = pendingCommitMap.get(tenantId); if (current != null && current.getTxId().equals(txId)) { try { - if (commitRequest.hasAddMsg()) { - addToCommit(ctx, current, commitRequest.getAddMsg()); - } else if (commitRequest.hasDeleteMsg()) { - deleteFromCommit(ctx, current, commitRequest.getDeleteMsg()); - } else if (commitRequest.hasPushMsg()) { + if (request.hasAddMsg()) { + addToCommit(ctx, current, request.getAddMsg()); + } else if (request.hasDeleteMsg()) { + deleteFromCommit(ctx, current, request.getDeleteMsg()); + } else if (request.hasPushMsg()) { reply(ctx, vcService.push(current)); } } catch (Exception e) { @@ -198,7 +273,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe throw e; } } else { - log.debug("[{}] Ignore request due to stale commit: {}", txId, commitRequest); + log.debug("[{}] Ignore request due to stale commit: {}", txId, request); } } } @@ -290,7 +365,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); } ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); - log.trace("PUSHING msg: {} to: {}", msg, tpi); + log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } @@ -304,9 +379,15 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } } + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + private Lock getRepoLock(TenantId tenantId) { return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); } - - } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 0d55783a31..758641f6eb 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -160,6 +160,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { //TODO: implement; } + @Override + public void fetch(TenantId tenantId) { + //Fetch latest changes on demand. + } + @Override public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { GitRepository repository = checkRepository(tenantId); @@ -197,9 +202,8 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + public List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception { GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); return repository.listFilesAtCommit(versionId, path).stream() .map(filePath -> { EntityId entityId = fromRelativePath(filePath); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index c748147856..54ab91582e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -30,7 +30,7 @@ public interface GitRepositoryService { List listVersions(TenantId tenantId, String branch, String path) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; @@ -51,4 +51,6 @@ public interface GitRepositoryService { List listBranches(TenantId tenantId); String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + + void fetch(TenantId tenantId); } From a5ff23a0a479275f35f4e443497f78daf0eec4d0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 24 May 2022 14:03:53 +0300 Subject: [PATCH 060/156] Add commit timestamp and timestamp sorting --- .../EntitiesVersionControlController.java | 29 +++++++++++----- .../DefaultGitVersionControlQueueService.java | 4 +-- common/cluster-api/src/main/proto/queue.proto | 11 ++++--- .../common/data/sync/vc/EntityVersion.java | 1 + .../DefaultClusterVersionControlService.java | 3 +- .../sync/vc/DefaultGitRepositoryService.java | 2 +- .../server/service/sync/vc/GitRepository.java | 33 +++++++++++++++---- 7 files changed, 58 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index ef1d402c42..0efa52dee2 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -53,9 +53,8 @@ import java.util.ArrayList; import java.util.List; import java.util.UUID; -import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.*; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; @RestController @TbCoreComponent @@ -131,10 +130,14 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink)); } catch (Exception e) { throw handleException(e); @@ -154,9 +157,13 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink)); } catch (Exception e) { throw handleException(e); @@ -183,9 +190,13 @@ public class EntitiesVersionControlController extends BaseController { @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) - @RequestParam int page) throws ThingsboardException { + @RequestParam int page, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { try { - PageLink pageLink = new PageLink(pageSize, page); + PageLink pageLink = createPageLink(pageSize, page, null, sortProperty, sortOrder); return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink)); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index fb6ab3e0d6..fbee0502e2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -312,7 +312,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getCommitId(), commitResponse.getName())); + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName())); commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); @@ -344,7 +344,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { - return new EntityVersion(proto.getId(), proto.getName()); + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName()); } private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index bd8e220a18..a1cbddf6e8 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -689,11 +689,12 @@ message CommitRequestMsg { } message CommitResponseMsg { - string commitId = 1; - string name = 2; - int32 added = 3; - int32 modified = 4; - int32 removed = 5; + int64 ts = 1; + string commitId = 2; + string name = 3; + int32 added = 4; + int32 modified = 5; + int32 removed = 6; } message PrepareMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 8779d64ed3..50d3fef8a8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -23,6 +23,7 @@ import lombok.NoArgsConstructor; @AllArgsConstructor @NoArgsConstructor public class EntityVersion { + private long timestamp; private String id; private String name; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index a9b821faaf..47d0aca292 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -245,7 +245,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTotalElements(data.getTotalElements()) .setHasNext(data.hasNext()) .addAllVersions(data.getData().stream().map( - v -> EntityVersionProto.newBuilder().setId(v.getId()).setName(v.getName()).build() + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).build() ).collect(Collectors.toList()))) ); } @@ -364,6 +364,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(CommitResponseMsg.newBuilder() + .setTs(result.getVersion().getTimestamp()) .setCommitId(result.getVersion().getId()) .setName(result.getVersion().getName()) .setAdded(result.getAdded()) diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index f51964f962..8d711c3e69 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -259,7 +259,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); } private EntityId fromRelativePath(String path) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index dcefe7bed7..6208b913f4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; import com.google.common.collect.Streams; import lombok.Data; import lombok.Getter; @@ -48,6 +49,7 @@ import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; @@ -58,10 +60,7 @@ import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.PublicKey; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; @@ -180,7 +179,7 @@ public class GitRepository { command.addPath(path); } Iterable commits = execute(command); - return iterableToPageData(commits, this::toCommit, pageLink); + return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); } public List listFilesAtCommit(String commitId) throws IOException { @@ -282,7 +281,7 @@ public class GitRepository { // } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { @@ -300,11 +299,30 @@ public class GitRepository { return command.call(); } - private static PageData iterableToPageData (Iterable iterable, Function mapper, PageLink pageLink) { + private static Function> revCommitComparatorFunction = pageLink -> { + SortOrder sortOrder = pageLink.getSortOrder(); + if (sortOrder != null + && sortOrder.getProperty().equals("timestamp") + && SortOrder.Direction.ASC.equals(sortOrder.getDirection())) { + return Comparator.comparingInt(RevCommit::getCommitTime); + } + return null; + }; + + private static PageData iterableToPageData (Iterable iterable, + Function mapper, + PageLink pageLink, + Function> comparatorFunction) { int totalElements = Iterables.size(iterable); int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; int startIndex = pageLink.getPageSize() * pageLink.getPage(); int limit = startIndex + pageLink.getPageSize(); + if (comparatorFunction != null) { + Comparator comparator = comparatorFunction.apply(pageLink); + if (comparator != null) { + iterable = Ordering.from(comparator).immutableSortedCopy(iterable); + } + } iterable = Iterables.limit(iterable, limit); if (startIndex < totalElements) { iterable = Iterables.skip(iterable, startIndex); @@ -374,6 +392,7 @@ public class GitRepository { @Data public static class Commit { + private final long timestamp; private final String id; private final String message; private final String authorName; From 9deb67afe7ecddfc7ad72c89430c1a6cb959cde1 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 24 May 2022 17:32:29 +0300 Subject: [PATCH 061/156] Version Control microservice --- .../src/main/resources/thingsboard.yml | 8 +- .../queue/discovery/ZkDiscoveryService.java | 2 - .../queue/settings/TbQueueCoreSettings.java | 2 + .../TbQueueRemoteJsInvokeSettings.java | 2 + .../settings/TbQueueRuleEngineSettings.java | 2 + .../settings/TbQueueTransportApiSettings.java | 2 + .../TbQueueTransportNotificationSettings.java | 2 + .../TbQueueVersionControlSettings.java | 2 + .../DefaultClusterVersionControlService.java | 156 ++++++++++---- .../sync/vc/DefaultGitRepositoryService.java | 37 ++-- .../service/sync/vc/GitRepositoryService.java | 3 + msa/pom.xml | 1 + msa/vc-executor-docker/docker/Dockerfile | 33 +++ .../docker/start-tb-vc-executor.sh | 33 +++ msa/vc-executor-docker/pom.xml | 190 ++++++++++++++++++ msa/vc-executor/pom.xml | 6 +- ...oardVersionControlExecutorApplication.java | 6 +- ...VersionControlQueueRoutingInfoService.java | 31 +++ ...ersionControlTenantRoutingInfoService.java | 30 +++ .../src/main/resources/logback.xml | 1 + .../src/main/resources/tb-vc-executor.yml | 160 ++++++++++++++- 21 files changed, 632 insertions(+), 77 deletions(-) create mode 100644 msa/vc-executor-docker/docker/Dockerfile create mode 100755 msa/vc-executor-docker/docker/start-tb-vc-executor.sh create mode 100644 msa/vc-executor-docker/pom.xml create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java create mode 100644 msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 76cc62d09a..aaec0852ca 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1025,7 +1025,7 @@ queue: topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1120,9 +1120,11 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" vc: - thread_pool_size: "${TB_VC_POOL_SIZE:4}" + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" git: - repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" management: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 8afd4dc768..16af4befd4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,14 +33,12 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 33e6552eb9..f64143b14c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueCoreSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index b930d04312..f4d62bad1d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRemoteJsInvokeSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index eee70e9e22..57131f6b62 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 0ad9414bd8..a4dfe86bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportApiSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index d4c0064692..e2a10b31b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportNotificationSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java index e7f0ce2259..3a7a5602d5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueVersionControlSettings { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index a9b821faaf..c4bb637eb0 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -15,8 +15,14 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; @@ -57,9 +63,10 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; -import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbVersionControlComponent; @@ -67,6 +74,7 @@ import org.thingsboard.server.queue.util.TbVersionControlComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -76,6 +84,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -87,7 +97,8 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { - private final TbServiceInfoProvider serviceInfoProvider; + private final PartitionService partitionService; + private final TbQueueProducerProvider producerProvider; private final TbVersionControlQueueFactory queueFactory; private final DataDecodingEncodingService encodingService; private final GitRepositoryService vcService; @@ -105,11 +116,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private long pollDuration; @Value("${queue.vc.pack-processing-timeout:60000}") private long packProcessingTimeout; + @Value("${vc.git.io_pool_size:3}") + private int ioPoolSize; + + //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. + private final List ioThreads = new ArrayList<>(); + @PostConstruct public void init() { consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); - producer = queueFactory.createTbCoreNotificationsMsgProducer(); + var threadFactory = ThingsBoardThreadFactory.forName("vc-io-thread"); + for (int i = 0; i < ioPoolSize; i++) { + ioThreads.add(MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(threadFactory))); + } + producer = producerProvider.getTbCoreNotificationsMsgProducer(); consumer = queueFactory.createToVersionControlMsgConsumer(); } @@ -122,11 +143,25 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe if (consumerExecutor != null) { consumerExecutor.shutdownNow(); } + ioThreads.forEach(ExecutorService::shutdownNow); } @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - //TODO: cleanup repositories that we no longer manage in this node. + for (TenantId tenantId : vcService.getActiveRepositoryTenants()) { + if (!partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId).isMyPartition()) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + pendingCommitMap.remove(tenantId); + vcService.clearRepository(tenantId); + } catch (Exception e) { + log.warn("[{}] Failed to cleanup the tenant repository", tenantId, e); + } finally { + lock.unlock(); + } + } + } consumer.subscribe(event.getPartitions()); } @@ -143,6 +178,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe void consumerLoop(TbQueueConsumer> consumer) { while (!stopped && !consumer.isStopped()) { + List> futures = new ArrayList<>(); try { List> msgs = consumer.poll(pollDuration); if (msgs.isEmpty()) { @@ -151,46 +187,17 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe for (TbProtoQueueMsg msgWrapper : msgs) { ToVersionControlServiceMsg msg = msgWrapper.getValue(); var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); - var lock = getRepoLock(ctx.getTenantId()); - lock.lock(); - try { - if (msg.hasClearRepositoryRequest()) { - handleClearRepositoryCommand(ctx); - } else { - if (msg.hasTestRepositoryRequest()) { - handleTestRepositoryCommand(ctx); - } else if (msg.hasInitRepositoryRequest()) { - handleInitRepositoryCommand(ctx); - } else { - var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); - var newSettings = ctx.getSettings(); - if (!newSettings.equals(currentSettings)) { - vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); - } else { - vcService.fetch(ctx.getTenantId()); - } - if (msg.hasCommitRequest()) { - handleCommitRequest(ctx, msg.getCommitRequest()); - } else if (msg.hasListBranchesRequest()) { - handleListBranches(ctx, msg.getListBranchesRequest()); - } else if (msg.hasListEntitiesRequest()) { - handleListEntities(ctx, msg.getListEntitiesRequest()); - } else if (msg.hasListVersionRequest()) { - handleListVersions(ctx, msg.getListVersionRequest()); - } else if (msg.hasEntityContentRequest()) { - handleEntityContentRequest(ctx, msg.getEntityContentRequest()); - } else if (msg.hasEntitiesContentRequest()) { - handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); - } - } - } - } catch (Exception e) { - reply(ctx, Optional.of(e)); - } finally { - lock.unlock(); - } + long startTs = System.currentTimeMillis(); + log.trace("[{}][{}] Submitting task.", ctx.getTenantId(), ctx.getRequestId()); + ListenableFuture future = ioThreads.get(ctx.getTenantId().hashCode() % ioPoolSize).submit(() -> processMessage(ctx, msg)); + logTaskExecution(ctx, future, startTs); + futures.add(future); + } + try { + Futures.allAsList(futures).get(packProcessingTimeout, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.info("Timeout for processing the version control tasks.", e); } - //TODO: handle timeouts and async processing for multiple tenants; consumer.commit(); } catch (Exception e) { if (!stopped) { @@ -206,6 +213,51 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe log.info("TB Version Control request consumer stopped."); } + private Void processMessage(VersionControlRequestCtx ctx, ToVersionControlServiceMsg msg) { + var lock = getRepoLock(ctx.getTenantId()); + lock.lock(); + try { + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(ctx); + } else { + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } else { + var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); + var newSettings = ctx.getSettings(); + if (!newSettings.equals(currentSettings)) { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } + if (msg.hasCommitRequest()) { + handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + vcService.fetch(ctx.getTenantId()); + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + vcService.fetch(ctx.getTenantId()); + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } + } + } + } catch (Exception e) { + reply(ctx, Optional.of(e)); + } finally { + lock.unlock(); + } + return null; + } + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { var entityType = EntityType.valueOf(request.getEntityType()); String path = getRelativePath(entityType, null); @@ -273,6 +325,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); if (request.hasPrepareMsg()) { + vcService.fetch(ctx.getTenantId()); prepareCommit(ctx, txId, request.getPrepareMsg()); } else if (request.hasAbortMsg()) { PendingCommit current = pendingCommitMap.get(tenantId); @@ -414,4 +467,21 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private Lock getRepoLock(TenantId tenantId) { return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); } + + private void logTaskExecution(VersionControlRequestCtx ctx, ListenableFuture future, long startTs) { + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback() { + + @Override + public void onSuccess(@Nullable Object result) { + log.trace("[{}][{}] Task processing took: {}ms", ctx.getTenantId(), ctx.getRequestId(), (System.currentTimeMillis() - startTs)); + } + + @Override + public void onFailure(Throwable t) { + log.trace("[{}][{}] Task failed: ", ctx.getTenantId(), ctx.getRequestId(), t); + } + }, MoreExecutors.directExecutor()); + } + } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index f51964f962..e30dd8aa20 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -35,19 +35,17 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -61,10 +59,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") private String repositoriesFolder; - @Value("${vc.git.repos-poll-interval:60}") - private long reposPollInterval; - - private ScheduledExecutorService scheduler; private final Map repositories = new ConcurrentHashMap<>(); @PostConstruct @@ -72,24 +66,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { if (StringUtils.isEmpty(repositoriesFolder)) { repositoriesFolder = defaultFolder; } - scheduler = Executors.newSingleThreadScheduledExecutor(); - scheduler.scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); } - @PreDestroy - public void stop() { - if (scheduler != null) { - scheduler.shutdownNow(); - } + @Override + public Set getActiveRepositoryTenants() { + return new HashSet<>(repositories.keySet()); } @Override @@ -151,6 +132,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @SneakyThrows @Override public void cleanUp(PendingCommit commit) { + log.debug("[{}] Cleanup tenant repository started.", commit.getTenantId()); GitRepository repository = checkRepository(commit.getTenantId()); try { repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); @@ -161,6 +143,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } repository.resetAndClean(); repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + log.debug("[{}] Cleanup tenant repository completed.", commit.getTenantId()); } @Override @@ -172,7 +155,9 @@ public class DefaultGitRepositoryService implements GitRepositoryService { public void fetch(TenantId tenantId) throws GitAPIException { var repository = repositories.get(tenantId); if (repository != null) { + log.debug("[{}] Fetching tenant repository.", tenantId); repository.fetch(); + log.debug("[{}] Fetched tenant repository.", tenantId); } } @@ -232,6 +217,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { @Override public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { clearRepository(tenantId); + log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository repository; if (Files.exists(repositoryDirectory)) { @@ -241,6 +227,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { Files.createDirectories(repositoryDirectory); repository = GitRepository.clone(settings, repositoryDirectory.toFile()); repositories.put(tenantId, repository); + log.debug("[{}] Init tenant repository completed.", tenantId); } @Override @@ -253,8 +240,10 @@ public class DefaultGitRepositoryService implements GitRepositoryService { public void clearRepository(TenantId tenantId) throws IOException { GitRepository repository = repositories.get(tenantId); if (repository != null) { + log.debug("[{}] Clear tenant repository started.", tenantId); FileUtils.deleteDirectory(new File(repository.getDirectory())); repositories.remove(tenantId); + log.debug("[{}] Clear tenant repository completed.", tenantId); } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 075e08a33e..9d57eee3ec 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -26,9 +26,12 @@ import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.io.IOException; import java.util.List; +import java.util.Set; public interface GitRepositoryService { + Set getActiveRepositoryTenants(); + void prepareCommit(PendingCommit pendingCommit); PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; diff --git a/msa/pom.xml b/msa/pom.xml index 888291502b..dd40348bc8 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -41,6 +41,7 @@ tb vc-executor + vc-executor-docker js-executor web-ui tb-node diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile new file mode 100644 index 0000000000..c2f9640a6b --- /dev/null +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -0,0 +1,33 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk11 + +COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-vc-executor.sh /usr/bin + +RUN yes | dpkg -i /tmp/${pkg.name}.deb +RUN rm /tmp/${pkg.name}.deb + +RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : + +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-vc-executor.sh"] diff --git a/msa/vc-executor-docker/docker/start-tb-vc-executor.sh b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh new file mode 100755 index 0000000000..4384008fd3 --- /dev/null +++ b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER="/config" +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +export LOADER_PATH=/config,${LOADER_PATH} + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=/config/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml new file mode 100644 index 0000000000..39fc35f32a --- /dev/null +++ b/msa/vc-executor-docker/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor-docker + pom + + ThingsBoard Version Control Executor Microservice + https://thingsboard.io + ThingsBoard Version Control Executor Microservice + + + UTF-8 + ${basedir}/../.. + tb-vc-executor + tb-vc-executor + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard.msa + vc-executor + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-tb-vc-executor-deb + package + + copy + + + + + org.thingsboard.msa + vc-executor + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml index 7e4df36117..0125354371 100644 --- a/msa/vc-executor/pom.xml +++ b/msa/vc-executor/pom.xml @@ -38,7 +38,7 @@ false process-resources package - tb-mqtt-transport + tb-vc-executor false ${project.build.directory}/windows ThingsBoard Version Control Executor Service @@ -46,6 +46,10 @@ + + org.thingsboard.common + queue + org.thingsboard.common version-control diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java index d30818110d..6978f94b51 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -15,17 +15,17 @@ package org.thingsboard.server.vc; /** */ import org.springframework.boot.SpringApplication; -import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; import java.util.Arrays; -@SpringBootConfiguration +@SpringBootApplication @EnableAsync @EnableScheduling -@ComponentScan({"org.thingsboard.server.vc", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) +@ComponentScan({"org.thingsboard.server", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) public class ThingsboardVersionControlExecutorApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java new file mode 100644 index 0000000000..0f4ac69948 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class VersionControlQueueRoutingInfoService implements QueueRoutingInfoService { + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java new file mode 100644 index 0000000000..ebd7ed07b3 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class VersionControlTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + //This dummy implementation is ok since Version Control service does not produce any rule engine messages. + return new TenantRoutingInfo(tenantId, false, false); + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index 2834934ee0..572d093dde 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -25,6 +25,7 @@ + diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 1de47e2750..8c34a91e80 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -23,5 +23,163 @@ server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). - port: "${HTTP_BIND_PORT:8080}" + port: "${HTTP_BIND_PORT:8086}" +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +queue: + type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + - key: max.poll.records + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + ota: + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +service: + type: "${TB_SERVICE_TYPE:tb-vc-executor}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" \ No newline at end of file From dbec05a2c173ec1446170c2300b757b1e0d05819 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 10:39:48 +0300 Subject: [PATCH 062/156] Test fix --- .../service/cluster/routing/HashPartitionServiceTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java index 74738c361f..6aafa833b1 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java @@ -71,6 +71,8 @@ public class HashPartitionServiceTest { queueRoutingInfoService); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10); + ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); + ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() .setServiceId("tb-core-0") From ca3c95afca1e55290e82cbe4e1a213402c4c9a57 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 25 May 2022 12:26:23 +0300 Subject: [PATCH 063/156] UI: Implement entitiy versions table --- .../EntitiesVersionControlController.java | 7 +- .../DefaultGitVersionControlQueueService.java | 2 +- .../server/service/sync/vc/GitRepository.java | 3 +- ui-ngx/src/app/core/auth/auth.actions.ts | 11 +- ui-ngx/src/app/core/auth/auth.models.ts | 1 + ui-ngx/src/app/core/auth/auth.reducer.ts | 6 +- ui-ngx/src/app/core/auth/auth.selectors.ts | 5 + ui-ngx/src/app/core/auth/auth.service.ts | 12 +- .../http/entities-version-control.service.ts | 27 +- ui-ngx/src/app/core/services/menu.service.ts | 17 ++ .../home/components/home-components.module.ts | 13 +- .../vc/entity-versions-table.component.html | 81 ++++++ .../vc/entity-versions-table.component.scss | 100 +++++++ .../vc/entity-versions-table.component.ts | 244 ++++++++++++++++++ .../version-control-settings.component.html | 2 +- .../version-control-settings.component.scss | 3 + .../vc}/version-control-settings.component.ts | 53 ++-- .../vc/version-control.component.html | 25 ++ .../vc/version-control.component.scss | 18 ++ .../vc/version-control.component.ts | 61 +++++ .../home/pages/admin/admin-routing.module.ts | 4 +- .../modules/home/pages/admin/admin.module.ts | 4 +- ...sion-control-admin-settings.component.html | 18 ++ ...ersion-control-admin-settings.component.ts | 44 ++++ .../pages/device/device-tabs.component.html | 5 + .../modules/home/pages/home-pages.module.ts | 4 +- .../home/pages/vc/vc-routing.module.ts | 44 ++++ .../app/modules/home/pages/vc/vc.module.ts | 31 +++ .../vc/branch-autocomplete.component.html | 5 +- .../vc/branch-autocomplete.component.ts | 147 ++++++++--- ui-ngx/src/app/shared/models/asset.models.ts | 4 +- ui-ngx/src/app/shared/models/base-data.ts | 6 + .../src/app/shared/models/customer.model.ts | 3 +- .../src/app/shared/models/dashboard.models.ts | 4 +- ui-ngx/src/app/shared/models/device.models.ts | 6 +- .../app/shared/models/rule-chain.models.ts | 4 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 10 +- 38 files changed, 948 insertions(+), 87 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.html (98%) rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.scss (95%) rename ui-ngx/src/app/modules/home/{pages/admin => components/vc}/version-control-settings.component.ts (82%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts create mode 100644 ui-ngx/src/app/modules/home/pages/vc/vc.module.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 0efa52dee2..bafc93c19f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -294,11 +294,14 @@ public class EntitiesVersionControlController extends BaseController { String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); if (StringUtils.isNotEmpty(defaultBranch)) { - remoteBranches.remove(defaultBranch); infos.add(new BranchInfo(defaultBranch, true)); } - remoteBranches.forEach(branch -> infos.add(new BranchInfo(branch, false))); + remoteBranches.forEach(branch -> { + if (!branch.equals(defaultBranch)) { + infos.add(new BranchInfo(branch, false)); + } + }); return infos; }, MoreExecutors.directExecutor())); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index fbee0502e2..f738582bb3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -411,7 +411,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } if (vcSettings != null) { builder.setVcSettings(ByteString.copyFrom(encodingService.encode(vcSettings))); - } else { + } else if (request.requiresSettings()) { throw new RuntimeException("No entity version control settings provisioned!"); } return builder; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 6208b913f4..20ed03795d 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -170,7 +170,7 @@ public class GitRepository { public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); + return new PageData<>(); } LogCommand command = git.log() .add(branchId) @@ -313,6 +313,7 @@ public class GitRepository { Function mapper, PageLink pageLink, Function> comparatorFunction) { + iterable = Streams.stream(iterable).collect(Collectors.toList()); int totalElements = Iterables.size(iterable); int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; int startIndex = pageLink.getPageSize() * pageLink.getPage(); diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 6edbdcd5a2..872607b822 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -23,7 +23,8 @@ export enum AuthActionTypes { UNAUTHENTICATED = '[Auth] Unauthenticated', LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', - UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id' + UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', + UPDATE_HAS_VERSION_CONTROL = '[Auth] Change Has Version Control' } export class ActionAuthAuthenticated implements Action { @@ -54,5 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } +export class ActionAuthUpdateHasVersionControl implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_VERSION_CONTROL; + + constructor(readonly payload: { hasVersionControl: boolean }) {} +} + export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasVersionControl; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 9267d48e28..a69b783526 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,6 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; + hasVersionControl: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index be2712300d..9d9404524b 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -23,7 +23,8 @@ const emptyUserAuthState: AuthPayload = { userTokenAccessEnabled: false, forceFullscreen: false, allowedDashboardIds: [], - edgesSupportEnabled: false + edgesSupportEnabled: false, + hasVersionControl: false }; export const initialState: AuthState = { @@ -54,6 +55,9 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; + case AuthActionTypes.UPDATE_HAS_VERSION_CONTROL: + return { ...state, ...action.payload}; + default: return state; } diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index aaadf00ec3..7a099d5acd 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,6 +55,11 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); +export const selectHasVersionControl = createSelector( + selectAuthState, + (state: AuthState) => state.hasVersionControl +); + export function getCurrentAuthState(store: Store): AuthState { let state: AuthState; store.pipe(select(selectAuth), take(1)).subscribe( diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 6b129da6ad..6fe809955f 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -437,17 +437,27 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } + private loadHasVersionControl(authUser: AuthUser): Observable { + if (authUser.authority === Authority.TENANT_ADMIN) { + return this.http.get('/api/admin/vcSettings/exists', defaultHttpOptions()); + } else { + return of(false); + } + } + private loadSystemParams(authPayload: AuthPayload): Observable { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), + this.loadHasVersionControl(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled}; + const hasVersionControl: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasVersionControl}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index 231581a02f..f8668f2ecf 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -18,7 +18,12 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { Observable } from 'rxjs'; -import { BranchInfo, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { PageData } from '@shared/models/page/page-data'; +import { DeviceInfo } from '@shared/models/device.models'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; @Injectable({ providedIn: 'root' @@ -37,4 +42,24 @@ export class EntitiesVersionControlService { public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); } + + public listEntityVersions(pageLink: PageLink, branch: string, + externalEntityId: EntityId, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listEntityTypeVersions(pageLink: PageLink, branch: string, + entityType: EntityType, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listVersions(pageLink: PageLink, branch: string, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 56191ec948..ee81f007ee 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -343,6 +343,13 @@ export class MenuService { path: '/dashboards', icon: 'dashboards' }, + { + id: guid(), + name: 'version-control.version-control', + type: 'link', + path: '/vc', + icon: 'history' + }, { id: guid(), name: 'audit-log.audit-logs', @@ -492,6 +499,16 @@ export class MenuService { } ] }, + { + name: 'version-control.management', + places: [ + { + name: 'version-control.version-control', + icon: 'history', + path: '/vc' + } + ] + }, { name: 'audit-log.audit', places: [ diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 05067e78ee..49f1f57fd1 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -154,6 +154,9 @@ import { QueueFormComponent } from '@home/components/queue/queue-form.component' import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-export-dialog.component'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { VersionControlComponent } from '@home/components/vc/version-control.component'; +import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; @NgModule({ declarations: @@ -278,7 +281,10 @@ import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-exp DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VcEntityExportDialogComponent + VcEntityExportDialogComponent, + VersionControlSettingsComponent, + VersionControlComponent, + EntityVersionsTableComponent ], imports: [ CommonModule, @@ -397,7 +403,10 @@ import { VcEntityExportDialogComponent } from '@home/components/vc/vc-entity-exp DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VcEntityExportDialogComponent + VcEntityExportDialogComponent, + VersionControlSettingsComponent, + VersionControlComponent, + EntityVersionsTableComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html new file mode 100644 index 0000000000..b47c2fcc79 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -0,0 +1,81 @@ + +
+
+ +
+
+ {{(singleEntityMode ? 'version-control.entity-versions' : 'version-control.versions') | translate}} + + +
+ + +
+ +
+ + + {{ 'version-control.created-time' | translate }} + + {{ entityVersion.timestamp | date:'yyyy-MM-dd HH:mm:ss' }} + + + + {{ 'version-control.version-id' | translate }} + + {{ entityVersion.id }} + + + + {{ 'version-control.version-name' | translate }} + + {{ entityVersion.name }} + + + + +
+ {{ + singleEntityMode + ? 'version-control.no-entity-versions-text' + : 'version-control.no-versions-text' + }} +
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss new file mode 100644 index 0000000000..9017de60b8 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../scss/constants'; + +:host { + width: 100%; + height: 100%; + display: block; + .tb-entity-table { + .tb-entity-table-content { + width: 100%; + height: 100%; + background: #fff; + + .mat-toolbar-tools{ + min-height: auto; + } + + .title-container{ + overflow: hidden; + } + + .tb-entity-table-title { + padding-right: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .table-container { + overflow: auto; + } + + .tb-entity-table-info{ + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .button-widget-action{ + margin-left: auto; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + @media #{$mat-xs} { + .mat-toolbar { + height: auto; + min-height: 100px; + + .tb-entity-table-title{ + padding-bottom: 5px; + width: 100%; + } + } + } +} + +:host ::ng-deep { + .mat-sort-header-sorted .mat-sort-header-arrow { + opacity: 1 !important; + } + tb-branch-autocomplete { + mat-form-field { + font-size: 16px; + width: 200px; + + .mat-form-field-wrapper { + padding-bottom: 0; + } + + .mat-form-field-underline { + bottom: 0; + } + + @media #{$mat-xs} { + width: 100%; + + .mat-form-field-infix { + width: auto !important; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts new file mode 100644 index 0000000000..4fc33beafb --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -0,0 +1,244 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + Input, + OnDestroy, + OnInit, + ViewChild +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityId } from '@shared/models/id/entity-id'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; +import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, map, tap } from 'rxjs/operators'; +import { EntityVersion } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatSort } from '@angular/material/sort'; +import { ResizeObserver } from '@juggle/resize-observer'; +import { hidePageSizePixelValue } from '@shared/models/constants'; +import { Direction, SortOrder } from '@shared/models/page/sort-order'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; + +@Component({ + selector: 'tb-entity-versions-table', + templateUrl: './entity-versions-table.component.html', + styleUrls: ['./entity-versions-table.component.scss'] +}) +export class EntityVersionsTableComponent extends PageComponent implements OnInit, AfterViewInit, OnDestroy { + + @ViewChild('branchAutocompleteComponent') branchAutocompleteComponent: BranchAutocompleteComponent; + + @Input() + singleEntityMode = false; + + displayedColumns = ['timestamp', 'id', 'name']; + pageLink: PageLink; + dataSource: EntityVersionsDatasource; + hidePageSize = false; + + branch: string = null; + + activeValue = false; + dirtyValue = false; + externalEntityIdValue: EntityId; + + viewsInited = false; + + private componentResize$: ResizeObserver; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + if (this.viewsInited) { + this.initFromDefaultBranch(); + } + } + } + } + + @Input() + set externalEntityId(externalEntityId: EntityId) { + if (this.externalEntityIdValue !== externalEntityId) { + this.externalEntityIdValue = externalEntityId; + this.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.dirtyValue = true; + } + } + } + + @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private elementRef: ElementRef) { + super(store); + this.dirtyValue = !this.activeValue; + const sortOrder: SortOrder = { property: 'timestamp', direction: Direction.DESC }; + this.pageLink = new PageLink(10, 0, null, sortOrder); + this.dataSource = new EntityVersionsDatasource(this.entitiesVersionControlService); + } + + ngOnInit() { + this.componentResize$ = new ResizeObserver(() => { + const showHidePageSize = this.elementRef.nativeElement.offsetWidth < hidePageSizePixelValue; + if (showHidePageSize !== this.hidePageSize) { + this.hidePageSize = showHidePageSize; + this.cd.markForCheck(); + } + }); + this.componentResize$.observe(this.elementRef.nativeElement); + } + + ngOnDestroy() { + if (this.componentResize$) { + this.componentResize$.disconnect(); + } + } + + branchChanged(newBranch: string) { + this.branch = newBranch; + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } + + ngAfterViewInit() { + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); + merge(this.sort.sortChange, this.paginator.page) + .pipe( + tap(() => this.updateData()) + ) + .subscribe(); + this.viewsInited = true; + if (!this.singleEntityMode) { + this.initFromDefaultBranch(); + } + } + + vcExport($event: Event) { + if ($event) { + $event.stopPropagation(); + } + } + + private initFromDefaultBranch() { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(false, true); + } + + private updateData() { + this.pageLink.page = this.paginator.pageIndex; + this.pageLink.pageSize = this.paginator.pageSize; + this.pageLink.sortOrder.property = this.sort.active; + this.pageLink.sortOrder.direction = Direction[this.sort.direction.toUpperCase()]; + this.dataSource.loadEntityVersions(this.singleEntityMode, this.branch, this.externalEntityIdValue, this.pageLink); + } + + private resetSortAndFilter(update: boolean) { + this.branch = null; + this.pageLink.textSearch = null; + if (this.viewsInited) { + this.paginator.pageIndex = 0; + const sortable = this.sort.sortables.get('timestamp'); + this.sort.active = sortable.id; + this.sort.direction = 'desc'; + if (update) { + this.initFromDefaultBranch(); + } + } + } +} + +class EntityVersionsDatasource implements DataSource { + + private entityVersionsSubject = new BehaviorSubject([]); + private pageDataSubject = new BehaviorSubject>(emptyPageData()); + + public pageData$ = this.pageDataSubject.asObservable(); + + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} + + connect(collectionViewer: CollectionViewer): Observable> { + return this.entityVersionsSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entityVersionsSubject.complete(); + this.pageDataSubject.complete(); + } + + loadEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + const result = new ReplaySubject>(); + this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entityVersionsSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + } + ); + return result; + } + + fetchEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + if (!branch) { + return of(emptyPageData()); + } else { + if (singleEntityMode) { + if (externalEntityId) { + return this.entitiesVersionControlService.listEntityVersions(pageLink, branch, externalEntityId, {ignoreErrors: true}); + } else { + return of(emptyPageData()); + } + } else { + return this.entitiesVersionControlService.listVersions(pageLink, branch, {ignoreErrors: true}); + } + } + } + + isEmpty(): Observable { + return this.entityVersionsSubject.pipe( + map((entityVersions) => !entityVersions.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html similarity index 98% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html index b512bb96f5..b48162d06f 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html @@ -16,7 +16,7 @@ -->
- +
admin.git-repository-settings diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss similarity index 95% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss index ede3570e68..d1d55faf19 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss @@ -14,6 +14,9 @@ * limitations under the License. */ :host { + mat-card.vc-settings { + margin: 8px; + } .fields-group { padding: 0 16px 8px; margin-bottom: 10px; diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts similarity index 82% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts rename to ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts index c2626ec062..2970987be7 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { Component, OnInit } from '@angular/core'; +import { Component, Input, OnInit } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; +import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { AdminService } from '@core/http/admin.service'; import { @@ -30,13 +30,21 @@ import { ActionNotificationShow } from '@core/notification/notification.actions' import { TranslateService } from '@ngx-translate/core'; import { isNotEmptyStr } from '@core/utils'; import { DialogService } from '@core/services/dialog.service'; +import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; +import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; +import { selectHasVersionControl } from '@core/auth/auth.selectors'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; @Component({ selector: 'tb-version-control-settings', templateUrl: './version-control-settings.component.html', - styleUrls: ['./version-control-settings.component.scss', './settings-card.scss'] + styleUrls: ['./version-control-settings.component.scss', './../../pages/admin/settings-card.scss'] }) -export class VersionControlSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class VersionControlSettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; versionControlSettingsForm: FormGroup; settings: EntitiesVersionControlSettings = null; @@ -62,7 +70,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On ngOnInit() { this.versionControlSettingsForm = this.fb.group({ repositoryUri: [null, [Validators.required]], - defaultBranch: [null, []], + defaultBranch: ['main', []], authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], username: [null, []], password: [null, []], @@ -77,16 +85,29 @@ export class VersionControlSettingsComponent extends PageComponent implements On this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { this.updateValidators(false); }); - this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).subscribe( + this.store.pipe( + select(selectHasVersionControl), + mergeMap((hasVersionControl) => { + if (hasVersionControl) { + return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( (settings) => { this.settings = settings; - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - } else { - this.showChangePrivateKeyPassword = true; + if (this.settings != null) { + if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.versionControlSettingsForm.reset(this.settings); + this.updateValidators(false); } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); }); } @@ -112,6 +133,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On } this.versionControlSettingsForm.reset(this.settings); this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: true })); } ); } @@ -131,18 +153,15 @@ export class VersionControlSettingsComponent extends PageComponent implements On this.showChangePrivateKeyPassword = false; this.changePrivateKeyPassword = false; formDirective.resetForm(); - this.versionControlSettingsForm.reset({ authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); + this.versionControlSettingsForm.reset({ defaultBranch: 'main', authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: false })); } ); } }); } - confirmForm(): FormGroup { - return this.versionControlSettingsForm; - } - changePasswordChanged() { if (this.changePassword) { this.versionControlSettingsForm.get('password').patchValue(''); diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html new file mode 100644 index 0000000000..903460054b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -0,0 +1,25 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss new file mode 100644 index 0000000000..da8df4b469 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts new file mode 100644 index 0000000000..07bdbcba25 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -0,0 +1,61 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit, ViewChild } from '@angular/core'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasVersionControl } from '@core/auth/auth.selectors'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { FormGroup } from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-version-control', + templateUrl: './version-control.component.html', + styleUrls: ['./version-control.component.scss'] +}) +export class VersionControlComponent implements OnInit, HasConfirmForm { + + @ViewChild('versionControlSettingsComponent', {static: false}) versionControlSettingsComponent: VersionControlSettingsComponent; + + @Input() + detailsMode = false; + + @Input() + active = true; + + @Input() + singleEntityMode = false; + + @Input() + externalEntityId: EntityId; + + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); + + constructor(private store: Store) { + + } + + ngOnInit() { + + } + + confirmForm(): FormGroup { + return this.versionControlSettingsComponent?.versionControlSettingsForm; + } + +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index d95b55e5f4..958e7b1c9b 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,7 +33,7 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; -import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; +import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -226,7 +226,7 @@ const routes: Routes = [ }, { path: 'vc', - component: VersionControlSettingsComponent, + component: VersionControlAdminSettingsComponent, canDeactivate: [ConfirmOnExitGuard], data: { auth: [Authority.TENANT_ADMIN], diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index aab8d4d56f..51612270df 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,7 +29,7 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; -import { VersionControlSettingsComponent } from '@home/pages/admin/version-control-settings.component'; +import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; @NgModule({ declarations: @@ -43,7 +43,7 @@ import { VersionControlSettingsComponent } from '@home/pages/admin/version-contr HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, - VersionControlSettingsComponent + VersionControlAdminSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html new file mode 100644 index 0000000000..869067e1fb --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html @@ -0,0 +1,18 @@ + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts new file mode 100644 index 0000000000..a203f64426 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; + +@Component({ + selector: 'tb-version-control-admin-settings', + templateUrl: './version-control-admin-settings.component.html', + styleUrls: [] +}) +export class VersionControlAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('versionControlSettingsComponent') versionControlSettingsComponent: VersionControlSettingsComponent; + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.versionControlSettingsComponent?.versionControlSettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index 3bec9b906c..33aac9ba44 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -49,3 +49,8 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts index 576ab1c9b2..feea05d76a 100644 --- a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts @@ -36,6 +36,7 @@ import { DeviceProfileModule } from './device-profile/device-profile.module'; import { ApiUsageModule } from '@home/pages/api-usage/api-usage.module'; import { EdgeModule } from '@home/pages/edge/edge.module'; import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; +import { VcModule } from '@home/pages/vc/vc.module'; @NgModule({ exports: [ @@ -56,7 +57,8 @@ import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; AuditLogModule, ApiUsageModule, OtaUpdateModule, - UserModule + UserModule, + VcModule ], providers: [ { diff --git a/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts b/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts new file mode 100644 index 0000000000..6054aaa14d --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/vc/vc-routing.module.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { RouterModule, Routes } from '@angular/router'; +import { ConfirmOnExitGuard } from '@core/guards/confirm-on-exit.guard'; +import { Authority } from '@shared/models/authority.enum'; +import { VersionControlComponent } from '@home/components/vc/version-control.component'; + +const routes: Routes = [ + { + path: 'vc', + component: VersionControlComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'version-control.version-control', + breadcrumb: { + label: 'version-control.version-control', + icon: 'history' + } + } + } +]; + +@NgModule({ + imports: [RouterModule.forChild(routes)], + exports: [RouterModule], + providers: [] +}) +export class VcRoutingModule { } diff --git a/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts b/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts new file mode 100644 index 0000000000..4944356a9d --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/vc/vc.module.ts @@ -0,0 +1,31 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@shared/shared.module'; +import { VcRoutingModule } from '@home/pages/vc/vc-routing.module'; + +@NgModule({ + declarations: [ + ], + imports: [ + CommonModule, + SharedModule, + VcRoutingModule + ] +}) +export class VcModule { } diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html index 3a37d707ea..b189d70199 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - + {{ 'version-control.branch' | translate }} - + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts index 421710181a..e72235d89e 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -14,7 +14,17 @@ /// limitations under the License. /// -import { AfterViewInit, Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@angular/core'; +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, + forwardRef, + Input, + NgZone, + OnInit, + ViewChild +} from '@angular/core'; import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; import { Observable, of } from 'rxjs'; import { @@ -24,6 +34,7 @@ import { map, publishReplay, refCount, + share, switchMap, tap } from 'rxjs/operators'; @@ -32,6 +43,7 @@ import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { BranchInfo } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { isNotEmptyStr } from '@core/utils'; @Component({ selector: 'tb-branch-autocomplete', @@ -60,27 +72,48 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit this.requiredValue = coerceBooleanProperty(value); } + private disabledValue: boolean; + + get disabled(): boolean { + return this.disabledValue; + } + @Input() - disabled: boolean; + set disabled(value: boolean) { + this.disabledValue = coerceBooleanProperty(value); + if (this.disabledValue) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } @Input() selectDefaultBranch = true; + @Input() + selectionMode = false; + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; - filteredBranches: Observable>; + filteredBranches: Observable>; - branches: Observable>; + branches: Observable> = null; + + defaultBranch: BranchInfo = null; searchText = ''; private dirty = false; + private ignoreClosedPanel = false; + private propagateChange = (v: any) => { }; constructor(private store: Store, private entitiesVersionControlService: EntitiesVersionControlService, - private fb: FormBuilder) { + private fb: FormBuilder, + private zone: NgZone) { this.branchFormGroup = this.fb.group({ branch: [null, []] }); @@ -94,17 +127,36 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } ngOnInit() { - - this.branches = null; this.filteredBranches = this.branchFormGroup.get('branch').valueChanges .pipe( + tap((value: BranchInfo | string) => { + let modelValue: BranchInfo | null; + if (typeof value === 'string' || !value) { + if (!this.selectionMode && typeof value === 'string' && isNotEmptyStr(value)) { + modelValue = {name: value, default: false}; + } else { + modelValue = null; + } + } else { + modelValue = value; + } + this.updateView(modelValue); + }), + map(value => { + if (value) { + if (typeof value === 'string') { + return value; + } else { + return value.name; + } + } else { + return ''; + } + }), debounceTime(150), distinctUntilChanged(), - tap(value => { - this.updateView(value); - }), - map(value => value ? value : ''), - switchMap(branch => this.fetchBranches(branch)) + switchMap(name => this.fetchBranches(name)), + share() ); } @@ -113,24 +165,16 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; - if (this.disabled) { - this.branchFormGroup.disable({emitEvent: false}); - } else { - this.branchFormGroup.enable({emitEvent: false}); - } } - selectDefaultBranchIfNeeded(): void { - if (this.selectDefaultBranch && !this.modelValue) { - this.getBranches().subscribe( + selectDefaultBranchIfNeeded(ignoreLoading = true, force = false): void { + if ((this.selectDefaultBranch && !this.modelValue) || force) { + this.getBranches(ignoreLoading).subscribe( (data) => { - if (data && data.length) { - const defaultBranch = data.find(branch => branch.default); - if (defaultBranch) { - this.modelValue = defaultBranch.name; - this.branchFormGroup.get('branch').patchValue(this.modelValue, {emitEvent: false}); - this.propagateChange(this.modelValue); - } + if (this.defaultBranch || force) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); } } ); @@ -141,9 +185,9 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit this.searchText = ''; this.modelValue = value; if (value != null) { - this.branchFormGroup.get('branch').patchValue(value, {emitEvent: false}); + this.branchFormGroup.get('branch').patchValue({name: value}, {emitEvent: false}); } else { - this.branchFormGroup.get('branch').patchValue('', {emitEvent: false}); + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: false}); this.selectDefaultBranchIfNeeded(); } this.dirty = true; @@ -156,31 +200,53 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } } - updateView(value: string | null) { - if (this.modelValue !== value) { - this.modelValue = value; + onPanelClosed() { + if (this.ignoreClosedPanel) { + this.ignoreClosedPanel = false; + } else { + if (this.selectionMode && !this.branchFormGroup.get('branch').value && this.defaultBranch) { + this.zone.run(() => { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: true}); + }, 0); + } + } + } + + updateView(value: BranchInfo | null) { + if (this.modelValue !== value?.name) { + this.modelValue = value?.name; this.propagateChange(this.modelValue); } } - displayBranchFn(branch?: string): string | undefined { - return branch ? branch : undefined; + displayBranchFn(branch?: BranchInfo): string | undefined { + return branch ? branch.name : undefined; } - fetchBranches(searchText?: string): Observable> { + fetchBranches(searchText?: string): Observable> { this.searchText = searchText; return this.getBranches().pipe( - map(branches => branches.map(branch => branch.name).filter(branchName => { - return searchText ? branchName.toUpperCase().startsWith(searchText.toUpperCase()) : true; - })) + map(branches => { + let res = branches.filter(branch => { + return searchText ? branch.name.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (!this.selectionMode && isNotEmptyStr(searchText) && !res.find(b => b.name === searchText)) { + res = [{name: searchText, default: false}, ...res]; + } + return res; + } + ) ); } - getBranches(): Observable> { + getBranches(ignoreLoading = true): Observable> { if (!this.branches) { - const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading: true, ignoreErrors: true}); + const branchesObservable = this.entitiesVersionControlService.listBranches({ignoreLoading, ignoreErrors: true}); this.branches = branchesObservable.pipe( catchError(() => of([] as Array)), + tap((data) => { + this.defaultBranch = data.find(branch => branch.default); + }), publishReplay(1), refCount() ); @@ -189,6 +255,7 @@ export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit } clear() { + this.ignoreClosedPanel = true; this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); setTimeout(() => { this.branchInput.nativeElement.blur(); diff --git a/ui-ngx/src/app/shared/models/asset.models.ts b/ui-ngx/src/app/shared/models/asset.models.ts index d38aee4d96..115eea4610 100644 --- a/ui-ngx/src/app/shared/models/asset.models.ts +++ b/ui-ngx/src/app/shared/models/asset.models.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { AssetId } from './id/asset-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; -export interface Asset extends BaseData { +export interface Asset extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/base-data.ts b/ui-ngx/src/app/shared/models/base-data.ts index 3af9bb9059..7b3ea9a642 100644 --- a/ui-ngx/src/app/shared/models/base-data.ts +++ b/ui-ngx/src/app/shared/models/base-data.ts @@ -27,6 +27,12 @@ export interface BaseData { label?: string; } +export interface ExportableEntity { + createdTime?: number; + id?: T; + externalId?: T; +} + export function hasIdEquals(id1: HasId, id2: HasId): boolean { if (isDefinedAndNotNull(id1) && isDefinedAndNotNull(id2)) { return id1.id === id2.id; diff --git a/ui-ngx/src/app/shared/models/customer.model.ts b/ui-ngx/src/app/shared/models/customer.model.ts index 435edf263f..67c1429189 100644 --- a/ui-ngx/src/app/shared/models/customer.model.ts +++ b/ui-ngx/src/app/shared/models/customer.model.ts @@ -17,8 +17,9 @@ import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; import { TenantId } from './id/tenant-id'; +import { ExportableEntity } from '@shared/models/base-data'; -export interface Customer extends ContactBased { +export interface Customer extends ContactBased, ExportableEntity { tenantId: TenantId; title: string; additionalInfo?: any; diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index 4332e484f6..12e1c83ada 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DashboardId } from '@shared/models/id/dashboard-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { ShortCustomerInfo } from '@shared/models/customer.model'; @@ -23,7 +23,7 @@ import { Timewindow } from '@shared/models/time/time.models'; import { EntityAliases } from './alias.models'; import { Filters } from '@shared/models/query/query.models'; -export interface DashboardInfo extends BaseData { +export interface DashboardInfo extends BaseData, ExportableEntity { tenantId?: TenantId; title?: string; image?: string; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index f5fc25e69d..c4c29f5da6 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DeviceId } from './id/device-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; @@ -560,7 +560,7 @@ export interface DeviceProfileData { provisionConfiguration?: DeviceProvisionConfiguration; } -export interface DeviceProfile extends BaseData { +export interface DeviceProfile extends BaseData, ExportableEntity { tenantId?: TenantId; name: string; description?: string; @@ -685,7 +685,7 @@ export interface DeviceData { transportConfiguration: DeviceTransportConfiguration; } -export interface Device extends BaseData { +export interface Device extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/rule-chain.models.ts b/ui-ngx/src/app/shared/models/rule-chain.models.ts index 85d6047191..01f9d268c3 100644 --- a/ui-ngx/src/app/shared/models/rule-chain.models.ts +++ b/ui-ngx/src/app/shared/models/rule-chain.models.ts @@ -14,14 +14,14 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainId } from '@shared/models/id/rule-chain-id'; import { RuleNodeId } from '@shared/models/id/rule-node-id'; import { RuleNode, RuleNodeComponentDescriptor, RuleNodeType } from '@shared/models/rule-node.models'; import { ComponentType } from '@shared/models/component-descriptor.models'; -export interface RuleChain extends BaseData { +export interface RuleChain extends BaseData, ExportableEntity { tenantId: TenantId; name: string; firstRuleNodeId: RuleNodeId; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0c890ff9b4..d574a1b55f 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -43,6 +43,7 @@ export interface BranchInfo { } export interface EntityVersion { + timestamp: number; id: string; name: string; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c37ba685e0..d5bac68d48 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3109,6 +3109,8 @@ "json-value-required": "JSON value is required." }, "version-control": { + "version-control": "Version control", + "management": "Version control management", "branch": "Branch", "select-branch": "Select branch", "branch-required": "Branch is required", @@ -3118,7 +3120,13 @@ "version-name-required": "Version name is required", "export-entity-relations": "Export entity relations", "export-entity-version-result-message": "Entity exported with version '{{name}}' and commit id '{{commitId}}'.", - "export-to-git": "Export to Git" + "export-to-git": "Export to Git", + "entity-versions": "Entity versions", + "versions": "Versions", + "created-time": "Created time", + "version-id": "Version ID", + "no-entity-versions-text": "No entity versions found", + "no-versions-text": "No versions found" }, "widget": { "widget-library": "Widgets Library", From 4f2e1a25fdd772ed27dba9509dbec639dc46bd60 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 12:58:55 +0300 Subject: [PATCH 064/156] Docker compose scripts for TB Version Control Executor --- .../vc/EntitiesVersionControlService.java | 2 - .../queue/discovery/HashPartitionService.java | 2 +- .../DefaultClusterVersionControlService.java | 17 ++++--- docker/.env | 1 + docker/.gitignore | 1 + docker/docker-compose.confluent.yml | 6 +++ docker/docker-compose.kafka.yml | 10 ++++ docker/docker-compose.postgres.volumes.yml | 10 ++++ docker/docker-compose.pubsub.yml | 33 +++--------- docker/docker-compose.rabbitmq.yml | 32 +++--------- docker/docker-compose.service-bus.yml | 30 +++-------- docker/docker-compose.yml | 32 ++++++++++++ docker/docker-create-log-folders.sh | 2 + docker/tb-vc-executor.env | 2 + docker/tb-vc-executor/conf/logback.xml | 51 +++++++++++++++++++ .../tb-vc-executor/conf/tb-vc-executor.conf | 23 +++++++++ .../server/msa/ThingsBoardDbInstaller.java | 10 +++- msa/vc-executor-docker/docker/Dockerfile | 3 ++ 18 files changed, 181 insertions(+), 86 deletions(-) create mode 100644 docker/tb-vc-executor.env create mode 100644 docker/tb-vc-executor/conf/logback.xml create mode 100644 docker/tb-vc-executor/conf/tb-vc-executor.conf diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 566e093cb4..08b92db96a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -50,10 +50,8 @@ public interface EntitiesVersionControlService { ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; - ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index e2325be878..583e196e2a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -54,7 +54,7 @@ public class HashPartitionService implements PartitionService { private String coreTopic; @Value("${queue.core.partitions:100}") private Integer corePartitions; - @Value("${queue.vc.topic}") + @Value("${queue.vc.topic:tb_version_control}") private String vcTopic; @Value("${queue.vc.partitions:10}") private Integer vcPartitions; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 93cd812a6a..5fcb9985bd 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -188,8 +188,9 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe ToVersionControlServiceMsg msg = msgWrapper.getValue(); var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); long startTs = System.currentTimeMillis(); - log.trace("[{}][{}] Submitting task.", ctx.getTenantId(), ctx.getRequestId()); - ListenableFuture future = ioThreads.get(ctx.getTenantId().hashCode() % ioPoolSize).submit(() -> processMessage(ctx, msg)); + log.trace("[{}][{}] RECEIVED task: {}", ctx.getTenantId(), ctx.getRequestId(), msg); + int threadIdx = Math.abs(ctx.getTenantId().hashCode() % ioPoolSize); + ListenableFuture future = ioThreads.get(threadIdx).submit(() -> processMessage(ctx, msg)); logTaskExecution(ctx, future, startTs); futures.add(future); } @@ -435,13 +436,17 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); if (e.isPresent()) { + log.debug("[{}][{}] Failed to process task", ctx.getTenantId(), ctx.getRequestId(), e.get()); builder.setError(e.get().getMessage()); - } - if (enrichFunction != null) { - builder = enrichFunction.apply(builder); } else { - builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + if (enrichFunction != null) { + builder = enrichFunction.apply(builder); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + log.debug("[{}][{}] Processed task", ctx.getTenantId(), ctx.getRequestId()); } + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); diff --git a/docker/.env b/docker/.env index 11e44fdde8..af0f537603 100644 --- a/docker/.env +++ b/docker/.env @@ -10,6 +10,7 @@ HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport +TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor TB_VERSION=latest diff --git a/docker/.gitignore b/docker/.gitignore index eee422d573..9c4c778f28 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -5,4 +5,5 @@ tb-node/db/** tb-node/postgres/** tb-node/cassandra/** tb-transports/*/log +tb-vc-executor/log/** !.env diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 6983dd3d3e..3d5abd0abe 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -61,3 +61,9 @@ services: tb-snmp-transport: env_file: - queue-confluent.env + tb-vc-executor1: + env_file: + - queue-confluent.env + tb-vc-executor2: + env_file: + - queue-confluent.env diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 2184528e53..09c4554562 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -90,3 +90,13 @@ services: - queue-kafka.env depends_on: - kafka + tb-vc-executor1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-vc-executor2: + env_file: + - queue-kafka.env + depends_on: + - kafka \ No newline at end of file diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 704e9400ae..019e087c48 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -53,6 +53,13 @@ services: tb-snmp-transport: volumes: - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + volumes: postgres-db-volume: @@ -76,3 +83,6 @@ volumes: tb-snmp-transport-log-volume: external: true name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index 0364957ee6..c03132d730 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -23,59 +23,40 @@ services: tb-core1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-pubsub.env + tb-vc-executor2: + env_file: + - queue-pubsub.env + diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index 1eb37709e5..d1acc32014 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-rabbitmq.env + tb-vc-executor2: + env_file: + - queue-rabbitmq.env \ No newline at end of file diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml index c511658b31..6e39de0baa 100644 --- a/docker/docker-compose.service-bus.yml +++ b/docker/docker-compose.service-bus.yml @@ -23,57 +23,39 @@ services: tb-core1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-service-bus.env tb-lwm2m-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-service-bus.env + tb-vc-executor2: + env_file: + - queue-service-bus.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 82fbaeed74..1ba6eda32d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -257,6 +257,38 @@ services: - "8080" env_file: - tb-web-ui.env + tb-vc-executor1: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor1 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 + tb-vc-executor2: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor2 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 5af0f96377..ba945a19df 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -26,3 +26,5 @@ mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log + +mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log diff --git a/docker/tb-vc-executor.env b/docker/tb-vc-executor.env new file mode 100644 index 0000000000..f92e30b78f --- /dev/null +++ b/docker/tb-vc-executor.env @@ -0,0 +1,2 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 diff --git a/docker/tb-vc-executor/conf/logback.xml b/docker/tb-vc-executor/conf/logback.xml new file mode 100644 index 0000000000..dc95b3a885 --- /dev/null +++ b/docker/tb-vc-executor/conf/logback.xml @@ -0,0 +1,51 @@ + + + + + + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.log + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/docker/tb-vc-executor/conf/tb-vc-executor.conf b/docker/tb-vc-executor/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..f140e3fb76 --- /dev/null +++ b/docker/tb-vc-executor/conf/tb-vc-executor.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-vc-executor/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-vc-executor/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-vc-executor.out +export LOADER_PATH=/usr/share/tb-vc-executor/conf diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 417799f64f..de55a3afa3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -34,6 +34,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume"; private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; + private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; private final DockerComposeExecutor dockerCompose; @@ -44,6 +45,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final String tbHttpTransportLogVolume; private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; + private final String tbVcExecutorLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -61,6 +63,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME; tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; + tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -72,6 +75,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); + env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); dockerCompose.withEnv(env); } @@ -104,6 +108,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbSnmpTransportLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); + dockerCompose.invokeDocker(); + dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); @@ -126,10 +133,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/"); copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); + copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); } diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile index c2f9640a6b..f4ff25e0e5 100644 --- a/msa/vc-executor-docker/docker/Dockerfile +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -18,6 +18,9 @@ FROM thingsboard/openjdk11 COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ +RUN mkdir -p /home/thingsboard/.config/jgit +RUN chown -R ${pkg.user}:${pkg.user} /home/thingsboard + RUN chmod a+x /tmp/*.sh \ && mv /tmp/start-tb-vc-executor.sh /usr/bin From 5bb9bdface4f9b03c2c9690c6c7e478b32bc05b5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 25 May 2022 13:43:35 +0300 Subject: [PATCH 065/156] Cache for VersionControlSettings --- .../service/session/SessionRedisCache.java | 2 +- .../DefaultEntitiesVersionControlService.java | 66 ++----------- ...efaultTbVersionControlSettingsService.java | 98 +++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 2 - .../vc/TbVersionControlSettingsService.java | 31 ++++++ .../VersionControlSettingsCaffeineCache.java | 36 +++++++ .../vc/VersionControlSettingsRedisCache.java | 41 ++++++++ .../src/main/resources/thingsboard.yml | 3 + .../server/common/data/CacheConstants.java | 1 + 9 files changed, 218 insertions(+), 62 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java diff --git a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java index cd3e8cb56b..dc12326bdd 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java @@ -33,7 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; public class SessionRedisCache extends RedisTbTransactionalCache { public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { @Override public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { return deviceSessionsCacheEntry.toByteArray(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7ad3e309e0..7f791163a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,9 +25,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -36,7 +34,6 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -46,7 +43,6 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -61,7 +57,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; @@ -86,10 +81,10 @@ import java.util.stream.Collectors; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final TbVersionControlSettingsService vcSettingsService; private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; - private final AdminSettingsService adminSettingsService; private final TransactionTemplate transactionTemplate; private ListeningExecutorService executor; @@ -311,54 +306,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; + return vcSettingsService.get(tenantId); } @Override public EntitiesVersionControlSettings saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - EntitiesVersionControlSettings storedSettings = null; - if (adminSettings != null) { - try { - storedSettings = JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - versionControlSettings = this.restoreCredentials(versionControlSettings, storedSettings); - if (adminSettings == null) { - adminSettings = new AdminSettings(); - adminSettings.setKey(SETTINGS_KEY); - adminSettings.setTenantId(tenantId); - } + versionControlSettings = this.vcSettingsService.restore(tenantId, versionControlSettings); try { //TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted. gitServiceQueue.initRepository(tenantId, versionControlSettings).get(); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); } - adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); - AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); - EntitiesVersionControlSettings savedVersionControlSettings; - try { - savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - return savedVersionControlSettings; + return vcSettingsService.save(tenantId, versionControlSettings); } @Override public void deleteVersionControlSettings(TenantId tenantId) throws Exception { - if (adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY)) { + if (vcSettingsService.delete(tenantId)) { //TODO: ashvayka: replace future.get with deferred result. Don't forget to call when tenant is deleted. gitServiceQueue.clearRepository(tenantId).get(); } @@ -366,8 +331,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { - EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); - settings = this.restoreCredentials(settings, storedSettings); + settings = this.vcSettingsService.restore(tenantId, settings); try { //TODO: ashvayka: replace future.get with deferred result. gitServiceQueue.testRepository(tenantId, settings).get(); @@ -379,7 +343,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private String getCauseMessage(Exception e) { String message; - if(e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())){ + if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) { message = e.getCause().getMessage(); } else { message = e.getMessage(); @@ -387,20 +351,4 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return message; } - private EntitiesVersionControlSettings restoreCredentials(EntitiesVersionControlSettings settings, EntitiesVersionControlSettings storedSettings) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { - if (storedSettings != null) { - settings.setPassword(storedSettings.getPassword()); - } - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { - if (storedSettings != null) { - settings.setPrivateKey(storedSettings.getPrivateKey()); - if (settings.getPrivateKeyPassword() == null) { - settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); - } - } - } - return settings; - } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java new file mode 100644 index 0000000000..a62b046ee2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DefaultTbVersionControlSettingsService implements TbVersionControlSettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + + @Override + public EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings settings) { + EntitiesVersionControlSettings storedSettings = get(tenantId); + if (storedSettings != null) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public EntitiesVersionControlSettings get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + } + return null; + }, true); + } + + @Override + public EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(SETTINGS_KEY); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + EntitiesVersionControlSettings savedVersionControlSettings; + try { + savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); + } catch (Exception e) { + throw new RuntimeException("Failed to load version control settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedVersionControlSettings; + } + + @Override + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 08b92db96a..1d8cfca00c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -36,8 +36,6 @@ import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { - String SETTINGS_KEY = "entitiesVersionControl"; - ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java new file mode 100644 index 0000000000..178499c6c7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; + +public interface TbVersionControlSettingsService { + + EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + EntitiesVersionControlSettings get(TenantId tenantId); + + EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java new file mode 100644 index 0000000000..8cab03ca29 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("VersionControlCache") +public class VersionControlSettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public VersionControlSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.VC_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java new file mode 100644 index 0000000000..9c778d6e4f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.gen.transport.TransportProtos; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("VersionControlCache") +public class VersionControlSettingsRedisCache extends RedisTbTransactionalCache { + + public VersionControlSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.VC_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index aaec0852ca..62c8052254 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -433,6 +433,9 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" + vcSettings: + timeToLiveInMinutes: "${CACHE_SPECS_VC_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_VC_SETTINGS_MAX_SIZE:10000}" redis: # standalone or cluster diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 571af34086..85ea116f0a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -31,4 +31,5 @@ public class CacheConstants { public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; + public static final String VC_SETTINGS_CACHE = "vcSettings"; } From ed44304eac4f779251912faf6a34349003f691f4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 25 May 2022 16:26:39 +0300 Subject: [PATCH 066/156] UI: Branch select improvements. Entity versions table improvements. --- .../DefaultGitVersionControlQueueService.java | 9 ++ common/cluster-api/src/main/proto/queue.proto | 2 + .../DefaultClusterVersionControlService.java | 11 +- .../server/service/sync/vc/GitRepository.java | 2 +- ui-ngx/src/app/core/http/admin.service.ts | 17 ++- .../http/entities-version-control.service.ts | 45 +++++- .../vc/entity-versions-table.component.html | 24 ++- .../vc/entity-versions-table.component.ts | 33 ++++- .../vc/branch-autocomplete.component.html | 8 +- .../vc/branch-autocomplete.component.ts | 140 ++++++++++-------- .../assets/locale/locale.constant-en_US.json | 3 +- 11 files changed, 210 insertions(+), 84 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index f738582bb3..66e0442297 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -150,6 +150,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setBranchName(branch) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } @@ -159,6 +162,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setBranchName(branch).setEntityType(entityType.name()) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } @@ -171,6 +177,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) .setPageSize(pageLink.getPageSize()) .setPage(pageLink.getPage()) + .setSortProperty(pageLink.getSortOrder() != null ? pageLink.getSortOrder().getProperty() : null) + .setSortDirection(pageLink.getSortOrder() != null && pageLink.getSortOrder().getDirection() != null + ? pageLink.getSortOrder().getDirection().name() : null) .build()); } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index a1cbddf6e8..82070a5895 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -724,6 +724,8 @@ message ListVersionsRequestMsg { int64 entityIdLSB = 4; int32 pageSize = 5; int32 page = 6; + string sortProperty = 7; + string sortDirection = 8; } message EntityVersionProto { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 93cd812a6a..a27a4862bf 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -290,7 +291,15 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } else { path = null; } - var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage())); + SortOrder sortOrder = null; + if (StringUtils.isNotEmpty(request.getSortProperty())) { + var direction = SortOrder.Direction.DESC; + if (StringUtils.isNotEmpty(request.getSortDirection())) { + direction = SortOrder.Direction.valueOf(request.getSortDirection()); + } + sortOrder = new SortOrder(request.getSortProperty(), direction); + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, new PageLink(request.getPageSize(), request.getPage(), null, sortOrder)); reply(ctx, Optional.empty(), builder -> builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() .setTotalPages(data.getTotalPages()) diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 20ed03795d..a1a46d5b25 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -281,7 +281,7 @@ public class GitRepository { // } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 79bb6580b8..7c12bd3a5d 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -26,6 +26,8 @@ import { TestSmsRequest, UpdateMessage } from '@shared/models/settings.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' @@ -33,7 +35,8 @@ import { export class AdminService { constructor( - private http: HttpClient + private http: HttpClient, + private entitiesVersionControlService: EntitiesVersionControlService ) { } public getAdminSettings(key: string, config?: RequestConfig): Observable> { @@ -72,11 +75,19 @@ export class AdminService { public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, config?: RequestConfig): Observable { return this.http.post('/api/admin/vcSettings', versionControlSettings, - defaultHttpOptionsFromConfig(config)); + defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); } public deleteEntitiesVersionControlSettings(config?: RequestConfig) { - return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)); + return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); } public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index f8668f2ecf..eba229de67 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -17,30 +17,63 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { Observable } from 'rxjs'; +import { combineLatest, Observable, of } from 'rxjs'; import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; -import { DeviceInfo } from '@shared/models/device.models'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; +import { createSelector, select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasVersionControl, selectIsAuthenticated, selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, combineAll, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' }) export class EntitiesVersionControlService { + branchList: Array = null; + constructor( - private http: HttpClient + private http: HttpClient, + private store: Store ) { + + this.store.pipe(select(selectIsUserLoaded)).subscribe( + () => { + this.branchList = null; + } + ); + } + + public clearBranchList(): void { + this.branchList = null; } - public listBranches(config?: RequestConfig): Observable> { - return this.http.get>('/api/entities/vc/branches', defaultHttpOptionsFromConfig(config)); + public listBranches(): Observable> { + if (!this.branchList) { + return this.http.get>('/api/entities/vc/branches', + defaultHttpOptionsFromConfig({ignoreErrors: true, ignoreLoading: false})).pipe( + catchError(() => of([] as Array)), + tap((list) => { + this.branchList = list; + }) + ); + } else { + return of(this.branchList); + } } public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { - return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)); + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + const branch = request.branch; + if (this.branchList && !this.branchList.find(b => b.name === branch)) { + this.branchList = null; + } + }) + ); } public listEntityVersions(pageLink: PageLink, branch: string, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b47c2fcc79..f5aa4a04d2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -48,27 +48,41 @@ - {{ 'version-control.version-id' | translate }} + {{ 'version-control.version-id' | translate }} - {{ entityVersion.id }} + + + - {{ 'version-control.version-name' | translate }} + {{ 'version-control.version-name' | translate }} {{ entityVersion.name }} - + - {{ singleEntityMode ? 'version-control.no-entity-versions-text' : 'version-control.no-versions-text' }} + {{ 'common.loading' | translate }}
7) { + versionId = versionId.slice(0, 7); + } + return versionId; + } + private initFromDefaultBranch() { - this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(false, true); + if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } else { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(true); + } } private updateData() { @@ -164,7 +182,6 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } private resetSortAndFilter(update: boolean) { - this.branch = null; this.pageLink.textSearch = null; if (this.viewsInited) { this.paginator.pageIndex = 0; @@ -185,6 +202,8 @@ class EntityVersionsDatasource implements DataSource { public pageData$ = this.pageDataSubject.asObservable(); + public dataLoading = true; + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} connect(collectionViewer: CollectionViewer): Observable> { @@ -199,6 +218,7 @@ class EntityVersionsDatasource implements DataSource { loadEntityVersions(singleEntityMode: boolean, branch: string, externalEntityId: EntityId, pageLink: PageLink): Observable> { + this.dataLoading = true; const result = new ReplaySubject>(); this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( catchError(() => of(emptyPageData())), @@ -207,6 +227,7 @@ class EntityVersionsDatasource implements DataSource { this.entityVersionsSubject.next(pageData.data); this.pageDataSubject.next(pageData); result.next(pageData); + this.dataLoading = false; } ); return result; diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html index b189d70199..74672b1772 100644 --- a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.html @@ -17,12 +17,14 @@ --> {{ 'version-control.branch' | translate }} - + [matAutocomplete]="branchAutocomplete"> - + +
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts new file mode 100644 index 0000000000..368a4bf441 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts @@ -0,0 +1,84 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; + +@Component({ + selector: 'tb-entity-version-export', + templateUrl: './entity-version-export.component.html', + styleUrls: [] +}) +export class EntityVersionExportComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + entityId: EntityId; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + exportFormGroup: FormGroup; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.exportFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + saveRelations: [false, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null, null); + } + } + + export(): void { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.exportFormGroup.get('branch').value, + versionName: this.exportFormGroup.get('versionName').value, + config: { + saveRelations: this.exportFormGroup.get('saveRelations').value + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (this.onClose) { + this.onClose(result, request.branch); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index f5aa4a04d2..11d8b2a2e2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -31,10 +31,12 @@ - diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss index 9017de60b8..ec5cb02256 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -78,7 +78,7 @@ tb-branch-autocomplete { mat-form-field { font-size: 16px; - width: 200px; + width: 250px; .mat-form-field-wrapper { padding-bottom: 0; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 884bab417b..436267fe2f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -21,8 +21,8 @@ import { ElementRef, Input, OnDestroy, - OnInit, - ViewChild + OnInit, Renderer2, + ViewChild, ViewContainerRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; @@ -33,7 +33,7 @@ import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, map, tap } from 'rxjs/operators'; -import { EntityVersion } from '@shared/models/vc.models'; +import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -42,6 +42,10 @@ import { hidePageSizePixelValue } from '@shared/models/constants'; import { Direction, SortOrder } from '@shared/models/page/sort-order'; import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; import { isNotEmptyStr } from '@core/utils'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-entity-versions-table', @@ -68,6 +72,8 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni viewsInited = false; + vcExportPopover: TbPopoverComponent; + private componentResize$: ResizeObserver; @Input() @@ -94,12 +100,18 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } + @Input() + entityId: EntityId; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, + private popoverService: TbPopoverService, + private renderer: Renderer2, private cd: ChangeDetectorRef, + private viewContainerRef: ViewContainerRef, private elementRef: ElementRef) { super(store); this.dirtyValue = !this.activeValue; @@ -148,10 +160,36 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } - vcExport($event: Event) { + toggleVcExport($event: Event, exportButton: MatButton) { if ($event) { $event.stopPropagation(); } + const trigger = exportButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + this.vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionExportComponent, 'bottom', true, null, + { + branch: this.branch, + entityId: this.entityId, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + this.vcExportPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + this.vcExportPopover.tbVisibleChange.subscribe((visible: boolean) => { + if (!visible) { + this.vcExportPopover = null; + } + }); + } } versionIdContent(entityVersion: EntityVersion): string { diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html deleted file mode 100644 index e7ea79cd73..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.html +++ /dev/null @@ -1,77 +0,0 @@ - -
- -

{{ (createResult ? 'version-control.entity-version-exported' : 'version-control.export-entity-version') | translate }}

- - - - - -
-
-
-
- - - - version-control.version-name - - - {{ 'version-control.version-name-required' | translate }} - - - - {{ 'version-control.export-entity-relations' | translate }} - -
-
-
-
-
-
-
-
- - -
-
- -
-
diff --git a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts b/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts deleted file mode 100644 index 9d3affea42..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/vc-entity-export-dialog.component.ts +++ /dev/null @@ -1,105 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; -import { ErrorStateMatcher } from '@angular/material/core'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm, Validators } from '@angular/forms'; -import { Router } from '@angular/router'; -import { DialogComponent } from '@app/shared/components/dialog.component'; -import { EntityId } from '@shared/models/id/entity-id'; -import { - SingleEntityVersionCreateRequest, - VersionCreateRequestType, - VersionCreationResult -} from '@shared/models/vc.models'; -import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; -import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; -import { TranslateService } from '@ngx-translate/core'; - -export interface VcEntityExportDialogData { - entityId: EntityId; -} - -@Component({ - selector: 'tb-vc-entity-export-dialog', - templateUrl: './vc-entity-export-dialog.component.html', - providers: [{provide: ErrorStateMatcher, useExisting: VcEntityExportDialogComponent}], - styleUrls: [] -}) -export class VcEntityExportDialogComponent extends DialogComponent - implements OnInit, ErrorStateMatcher { - - exportFormGroup: FormGroup; - - submitted = false; - - createResult: VersionCreationResult; - - createResultMessage: SafeHtml; - - constructor(protected store: Store, - protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: VcEntityExportDialogData, - @SkipSelf() private errorStateMatcher: ErrorStateMatcher, - public dialogRef: MatDialogRef, - private entitiesVersionControlService: EntitiesVersionControlService, - private translate: TranslateService, - private domSanitizer: DomSanitizer, - private fb: FormBuilder) { - super(store, router, dialogRef); - - this.exportFormGroup = this.fb.group({ - branch: [null, [Validators.required]], - versionName: [null, [Validators.required]], - saveRelations: [false, []] - }); - } - - ngOnInit(): void { - } - - isErrorState(control: FormControl | null, form: FormGroupDirective | NgForm | null): boolean { - const originalErrorState = this.errorStateMatcher.isErrorState(control, form); - const customErrorState = !!(control && control.invalid && this.submitted); - return originalErrorState || customErrorState; - } - - cancel(): void { - this.dialogRef.close(); - } - - export(): void { - this.submitted = true; - const request: SingleEntityVersionCreateRequest = { - entityId: this.data.entityId, - branch: this.exportFormGroup.get('branch').value, - versionName: this.exportFormGroup.get('versionName').value, - config: { - saveRelations: this.exportFormGroup.get('saveRelations').value - }, - type: VersionCreateRequestType.SINGLE_ENTITY - }; - this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - this.createResult = result; - const message = this.translate.instant('version-control.export-entity-version-result-message', - {name: result.version.name, commitId: result.version.id}); - this.createResultMessage = this.domSanitizer.bypassSecurityTrustHtml(message); - }); - } -} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index 903460054b..c631819128 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -21,5 +21,6 @@ diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts index 07bdbcba25..2ca68255ae 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -44,6 +44,9 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { @Input() externalEntityId: EntityId; + @Input() + entityId: EntityId; + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); constructor(private store: Store) { diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 1730c69084..ff08af8039 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -22,11 +22,6 @@ import { ImportDialogCsvComponent, ImportDialogCsvData } from '@home/components/import-export/import-dialog-csv.component'; -import { EntityId } from '@shared/models/id/entity-id'; -import { - VcEntityExportDialogComponent, - VcEntityExportDialogData -} from '@home/components/vc/vc-entity-export-dialog.component'; @Injectable() export class HomeDialogsService { @@ -46,17 +41,6 @@ export class HomeDialogsService { } } - public exportVcEntity(entityId: EntityId): Observable { - return this.dialog.open(VcEntityExportDialogComponent, - { - disableClose: true, - panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], - data: { - entityId - } - }).afterClosed(); - } - private openImportDialogCSV(entityType: EntityType, importTitle: string, importFileLabel: string): Observable { return this.dialog.open(ImportDialogCsvComponent, { diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index e5accc15a9..3a3f2dda92 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -49,3 +49,8 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html index 75171b4532..00ed882cd9 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset.component.html @@ -46,12 +46,6 @@ [fxShow]="!isEdit && assetScope === 'edge'"> {{ 'edge.unassign-from-edge' | translate }} - - - - - + + + + +
+ + +   + + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 436267fe2f..b60665f2b4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -29,10 +29,10 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntityId } from '@shared/models/id/entity-id'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; -import { BehaviorSubject, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; -import { catchError, map, tap } from 'rxjs/operators'; +import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; @@ -61,6 +61,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni displayedColumns = ['timestamp', 'id', 'name']; pageLink: PageLink; + textSearchMode = false; dataSource: EntityVersionsDatasource; hidePageSize = false; @@ -103,6 +104,8 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @ViewChild('searchInput') searchInputField: ElementRef; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @@ -148,6 +151,17 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } ngAfterViewInit() { + fromEvent(this.searchInputField.nativeElement, 'keyup') + .pipe( + debounceTime(400), + distinctUntilChanged(), + tap(() => { + this.paginator.pageIndex = 0; + this.updateData(); + }) + ) + .subscribe(); + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); merge(this.sort.sortChange, this.paginator.page) .pipe( @@ -155,7 +169,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni ) .subscribe(); this.viewsInited = true; - if (!this.singleEntityMode) { + if (!this.singleEntityMode || (this.activeValue && this.externalEntityIdValue)) { this.initFromDefaultBranch(); } } @@ -200,6 +214,22 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni return versionId; } + enterFilterMode() { + this.textSearchMode = true; + this.pageLink.textSearch = ''; + setTimeout(() => { + this.searchInputField.nativeElement.focus(); + this.searchInputField.nativeElement.setSelectionRange(0, 0); + }, 10); + } + + exitFilterMode() { + this.textSearchMode = false; + this.pageLink.textSearch = null; + this.paginator.pageIndex = 0; + this.updateData(); + } + private initFromDefaultBranch() { if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { this.paginator.pageIndex = 0; @@ -220,6 +250,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } private resetSortAndFilter(update: boolean) { + this.textSearchMode = false; this.pageLink.textSearch = null; if (this.viewsInited) { this.paginator.pageIndex = 0; diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts index 2970987be7..3462fbb728 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts @@ -33,7 +33,7 @@ import { DialogService } from '@core/services/dialog.service'; import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; import { selectHasVersionControl } from '@core/auth/auth.selectors'; -import { catchError, mergeMap } from 'rxjs/operators'; +import { catchError, mergeMap, take } from 'rxjs/operators'; import { of } from 'rxjs'; @Component({ @@ -87,6 +87,7 @@ export class VersionControlSettingsComponent extends PageComponent implements On }); this.store.pipe( select(selectHasVersionControl), + take(1), mergeMap((hasVersionControl) => { if (hasVersionControl) { return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( From 905b60da6a4a6d006cfc86b30861358cb2efd190 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 26 May 2022 12:32:51 +0300 Subject: [PATCH 071/156] Fix GitRepository.status command, Increased timeout between poll commands. --- application/src/main/resources/thingsboard.yml | 3 +++ .../thingsboard/server/service/sync/vc/GitRepository.java | 5 ++++- msa/vc-executor/src/main/resources/tb-vc-executor.yml | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 62c8052254..31e5d7a76e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -931,6 +931,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index a0c905852b..c22e18c004 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -241,7 +241,10 @@ public class GitRepository { public Status status() throws GitAPIException { org.eclipse.jgit.api.Status status = execute(git.status()); - return new Status(status.getAdded(), status.getModified(), status.getRemoved()); + Set modified = new HashSet<>(); + modified.addAll(status.getModified()); + modified.addAll(status.getChanged()); + return new Status(status.getAdded(), modified, status.getRemoved()); } public Commit commit(String message) throws GitAPIException { diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 8c34a91e80..4e430e5ba6 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -73,6 +73,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" From 46e0c7c90897db26fa30c604bfd7dc6b4aa5a2ef Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 13:50:50 +0300 Subject: [PATCH 072/156] UI: Implement entity version restore --- .../DefaultEntitiesVersionControlService.java | 2 +- .../http/entities-version-control.service.ts | 12 ++- .../home/components/home-components.module.ts | 7 +- .../vc/entity-version-export.component.html | 95 +++++++++++-------- .../vc/entity-version-export.component.scss | 21 ++++ .../vc/entity-version-export.component.ts | 16 +++- .../vc/entity-version-restore.component.html | 49 ++++++++++ .../vc/entity-version-restore.component.ts | 86 +++++++++++++++++ .../vc/entity-versions-table.component.html | 15 +++ .../vc/entity-versions-table.component.ts | 56 ++++++++--- .../vc/version-control.component.html | 3 +- .../vc/version-control.component.ts | 5 +- .../pages/asset/asset-tabs.component.html | 1 + .../customer/customer-tabs.component.html | 1 + .../dashboard/dashboard-tabs.component.html | 1 + .../device-profile-tabs.component.html | 3 +- .../pages/device/device-tabs.component.html | 1 + .../rulechain/rulechain-tabs.component.html | 1 + ui-ngx/src/app/shared/models/vc.models.ts | 29 ++++++ .../assets/locale/locale.constant-en_US.json | 9 +- 20 files changed, 347 insertions(+), 66 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 825171493e..e812595ca4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -190,7 +190,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - Futures.transform(future, entityData -> { + return Futures.transform(future, entityData -> { EntityImportResult importResult = transactionTemplate.execute(status -> { try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index eba229de67..86674f0569 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -18,7 +18,13 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; import { combineLatest, Observable, of } from 'rxjs'; -import { BranchInfo, EntityVersion, VersionCreateRequest, VersionCreationResult } from '@shared/models/vc.models'; +import { + BranchInfo, + EntityVersion, + VersionCreateRequest, + VersionCreationResult, + VersionLoadRequest, VersionLoadResult +} from '@shared/models/vc.models'; import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; @@ -95,4 +101,8 @@ export class EntitiesVersionControlService { return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, defaultHttpOptionsFromConfig(config)); } + + public loadEntitiesVersion(request: VersionLoadRequest, config?: RequestConfig): Observable> { + return this.http.post>('/api/entities/vc/entity', request, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 38cf85e72c..6134708176 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -157,6 +157,7 @@ import { VersionControlSettingsComponent } from '@home/components/vc/version-con import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; @NgModule({ declarations: @@ -284,7 +285,8 @@ import { EntityVersionExportComponent } from '@home/components/vc/entity-version VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent + EntityVersionExportComponent, + EntityVersionRestoreComponent ], imports: [ CommonModule, @@ -406,7 +408,8 @@ import { EntityVersionExportComponent } from '@home/components/vc/entity-version VersionControlSettingsComponent, VersionControlComponent, EntityVersionsTableComponent, - EntityVersionExportComponent + EntityVersionExportComponent, + EntityVersionRestoreComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html index 5791790b37..3838ea111d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html @@ -16,45 +16,58 @@ -->
- -

{{ 'version-control.create-entity-version' | translate }}

- - - - -
-
-
- - - - version-control.version-name - - - {{ 'version-control.version-name-required' | translate }} - - - - {{ 'version-control.export-entity-relations' | translate }} - -
-
-
-
- - -
+
+ +

{{ 'version-control.create-entity-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+
+ + +
+
+
+
{{ resultMessage }}
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss new file mode 100644 index 0000000000..46e55cf4d4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .export-result-message { + padding: 48px 8px 8px; + text-align: center; + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts index 368a4bf441..2aa3123858 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.ts @@ -26,11 +26,12 @@ import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; @Component({ selector: 'tb-entity-version-export', templateUrl: './entity-version-export.component.html', - styleUrls: [] + styleUrls: ['./entity-version-export.component.scss'] }) export class EntityVersionExportComponent extends PageComponent implements OnInit { @@ -43,10 +44,16 @@ export class EntityVersionExportComponent extends PageComponent implements OnIni @Input() onClose: (result: VersionCreationResult | null, branch: string | null) => void; + @Input() + onContentUpdated: () => void; + exportFormGroup: FormGroup; + resultMessage: string; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, private fb: FormBuilder) { super(store); } @@ -76,7 +83,12 @@ export class EntityVersionExportComponent extends PageComponent implements OnIni type: VersionCreateRequestType.SINGLE_ENTITY }; this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - if (this.onClose) { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + } else if (this.onClose) { this.onClose(result, request.branch); } }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html new file mode 100644 index 0000000000..cb71696b4e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -0,0 +1,49 @@ + +
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + +
+
+
+ + {{ 'version-control.load-entity-relations' | translate }} + +
+
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts new file mode 100644 index 0000000000..2cc615c011 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -0,0 +1,86 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { SingleEntityVersionLoadRequest, VersionLoadRequestType, VersionLoadResult } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; + +@Component({ + selector: 'tb-entity-version-restore', + templateUrl: './entity-version-restore.component.html', + styleUrls: [] +}) +export class EntityVersionRestoreComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: (result: Array | null) => void; + + restoreFormGroup: FormGroup; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.restoreFormGroup = this.fb.group({ + loadRelations: [false, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + restore(): void { + const request: SingleEntityVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + config: { + loadRelations: this.restoreFormGroup.get('loadRelations').value + }, + type: VersionLoadRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + if (this.onClose) { + this.onClose(result); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 5acfe3b0ba..2e5302881e 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -99,6 +99,21 @@ {{ entityVersion.name }} + + + + +
+ +
+ + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index b60665f2b4..06c778d278 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -18,10 +18,10 @@ import { AfterViewInit, ChangeDetectorRef, Component, - ElementRef, + ElementRef, EventEmitter, Input, OnDestroy, - OnInit, Renderer2, + OnInit, Output, Renderer2, ViewChild, ViewContainerRef } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; @@ -33,7 +33,7 @@ import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { PageLink } from '@shared/models/page/page-link'; import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; -import { EntityVersion, VersionCreationResult } from '@shared/models/vc.models'; +import { EntityVersion, VersionCreationResult, VersionLoadResult } from '@shared/models/vc.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; @@ -46,6 +46,7 @@ import { TbPopoverService } from '@shared/components/popover.service'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; import { MatButton } from '@angular/material/button'; import { TbPopoverComponent } from '@shared/components/popover.component'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; @Component({ selector: 'tb-entity-versions-table', @@ -59,7 +60,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() singleEntityMode = false; - displayedColumns = ['timestamp', 'id', 'name']; + displayedColumns = ['timestamp', 'id', 'name', 'actions']; pageLink: PageLink; textSearchMode = false; dataSource: EntityVersionsDatasource; @@ -73,8 +74,6 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni viewsInited = false; - vcExportPopover: TbPopoverComponent; - private componentResize$: ResizeObserver; @Input() @@ -104,6 +103,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @Output() + versionRestored = new EventEmitter(); + @ViewChild('searchInput') searchInputField: ElementRef; @ViewChild(MatPaginator) paginator: MatPaginator; @@ -182,13 +184,13 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni if (this.popoverService.hasPopover(trigger)) { this.popoverService.hidePopover(trigger); } else { - this.vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionExportComponent, 'bottom', true, null, + const vcExportPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionExportComponent, 'left', true, null, { branch: this.branch, entityId: this.entityId, onClose: (result: VersionCreationResult | null, branch: string | null) => { - this.vcExportPopover.hide(); + vcExportPopover.hide(); if (result) { if (this.branch !== branch) { this.branchChanged(branch); @@ -196,13 +198,39 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } + }, + onContentUpdated: () => { + vcExportPopover.updatePosition(); + setTimeout(() => { + vcExportPopover.updatePosition(); + }); + } + }, {}, {}, {}, false); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + externalEntityId: this.externalEntityIdValue, + onClose: (result: Array | null) => { + restoreVersionPopover.hide(); + if (result && result.length) { + this.versionRestored.emit(); + } } }, {}, {}, {}, false); - this.vcExportPopover.tbVisibleChange.subscribe((visible: boolean) => { - if (!visible) { - this.vcExportPopover = null; - } - }); } } diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index c631819128..e5e650ac85 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -22,5 +22,6 @@ + [externalEntityId]="externalEntityId" + (versionRestored)="versionRestored.emit()"> diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts index 2ca68255ae..da6828c4a0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit, ViewChild } from '@angular/core'; +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core'; import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { selectHasVersionControl } from '@core/auth/auth.selectors'; @@ -47,6 +47,9 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { @Input() entityId: EntityId; + @Output() + versionRestored = new EventEmitter(); + hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); constructor(private store: Store) { diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index 3a3f2dda92..c115a43ab6 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html index 7524fd59f7..634443a397 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html index 21b20bcb82..61a030ac16 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html @@ -22,5 +22,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html index e713e70690..effac9a7c8 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profile-tabs.component.html @@ -75,8 +75,9 @@ label="{{ 'audit-log.audit-logs' | translate }}"> - diff --git a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html index d84306d029..78d494d8ae 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/device-tabs.component.html @@ -52,5 +52,6 @@ diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html index c60910b565..49a2d83372 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-tabs.component.html @@ -55,5 +55,6 @@ diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index d574a1b55f..4975b8ea2b 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -15,16 +15,26 @@ /// import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; export interface VersionCreateConfig { saveRelations: boolean; } +export interface VersionLoadConfig { + loadRelations: boolean; +} + export enum VersionCreateRequestType { SINGLE_ENTITY = 'SINGLE_ENTITY', COMPLEX = 'COMPLEX' } +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + export interface VersionCreateRequest { versionName: string; branch: string; @@ -37,6 +47,18 @@ export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { type: VersionCreateRequestType.SINGLE_ENTITY; } +export interface VersionLoadRequest { + branch: string; + versionId: string; + type: VersionLoadRequestType; +} + +export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { + externalEntityId: EntityId; + config: VersionLoadConfig; + type: VersionLoadRequestType.SINGLE_ENTITY; +} + export interface BranchInfo { name: string; default: boolean; @@ -54,3 +76,10 @@ export interface VersionCreationResult { modified: number; removed: number; } + +export interface VersionLoadResult { + entityType: EntityType; + created: number; + updated: number; + deleted: number; +} diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 202597ade1..0a2f39bcd4 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -57,7 +57,8 @@ "download": "Download", "next-with-label": "Next: {{label}}", "read-more": "Read more", - "hide": "Hide" + "hide": "Hide", + "restore": "Restore" }, "aggregation": { "aggregation": "Aggregation", @@ -3126,7 +3127,11 @@ "no-entity-versions-text": "No entity versions found", "no-versions-text": "No versions found", "copy-full-version-id": "Copy full version id", - "create-version": "Create version" + "create-version": "Create version", + "nothing-to-commit": "Nothing to commit", + "restore-version": "Restore version", + "restore-entity-from-version": "Restore entity from version '{{versionName}}'", + "load-entity-relations": "Load entity relations" }, "widget": { "widget-library": "Widgets Library", From 6558b0b24d2d4a4baae5edaefab987cd1f38fe96 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 14:27:22 +0300 Subject: [PATCH 073/156] UI: Improve restore action --- .../home/components/vc/entity-versions-table.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 06c778d278..fcaa12f352 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -27,7 +27,7 @@ import { import { PageComponent } from '@shared/components/page.component'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { EntityId } from '@shared/models/id/entity-id'; +import { EntityId, entityIdEquals } from '@shared/models/id/entity-id'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; @@ -91,7 +91,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() set externalEntityId(externalEntityId: EntityId) { - if (this.externalEntityIdValue !== externalEntityId) { + if (!entityIdEquals(this.externalEntityIdValue, externalEntityId)) { this.externalEntityIdValue = externalEntityId; this.resetSortAndFilter(this.activeValue); if (!this.activeValue) { From c46b5b6448e81bca42bd4a7afde261ad6f030ac4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 26 May 2022 14:34:25 +0300 Subject: [PATCH 074/156] Fix message --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0a2f39bcd4..24cdff2735 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3128,7 +3128,7 @@ "no-versions-text": "No versions found", "copy-full-version-id": "Copy full version id", "create-version": "Create version", - "nothing-to-commit": "Nothing to commit", + "nothing-to-commit": "No changes to commit", "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-entity-relations": "Load entity relations" From e4ed8c508098849efdbef94834d7e31e3a3235e2 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 26 May 2022 15:10:46 +0300 Subject: [PATCH 075/156] Api for comparing current entity data to specific version; refactoring --- .../EntitiesVersionControlController.java | 13 +++ .../DefaultEntitiesExportImportService.java | 10 +- .../DefaultEntitiesVersionControlService.java | 32 +++++- .../DefaultGitVersionControlQueueService.java | 83 ++++++++++---- .../vc/EntitiesVersionControlService.java | 5 +- .../vc/GitVersionControlQueueService.java | 6 + .../{ => data}/ClearRepositoryGitRequest.java | 2 +- .../sync/vc/{ => data}/CommitGitRequest.java | 2 +- .../sync/vc/data/ContentsDiffGitRequest.java | 33 ++++++ .../{ => data}/EntitiesContentGitRequest.java | 3 +- .../{ => data}/EntityContentGitRequest.java | 2 +- .../vc/{ => data}/ListBranchesGitRequest.java | 3 +- .../vc/{ => data}/ListEntitiesGitRequest.java | 3 +- .../vc/{ => data}/ListVersionsGitRequest.java | 6 +- .../sync/vc/{ => data}/PendingGitRequest.java | 2 +- .../sync/vc/data/VersionsDiffGitRequest.java | 38 ++++++ .../sync/vc/{ => data}/VoidGitRequest.java | 5 +- common/cluster-api/src/main/proto/queue.proto | 32 ++++++ .../common/data/sync/vc/EntityDataDiff.java | 28 +++++ .../data/sync/vc/EntityVersionsDiff.java | 34 ++++++ .../load/EntityTypeVersionLoadConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 - .../thingsboard/common/util/JacksonUtil.java | 13 ++- .../DefaultClusterVersionControlService.java | 39 ++++++- .../sync/vc/DefaultGitRepositoryService.java | 21 ++-- .../server/service/sync/vc/GitRepository.java | 108 ++++++++++++------ .../service/sync/vc/GitRepositoryService.java | 5 + 27 files changed, 433 insertions(+), 97 deletions(-) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ClearRepositoryGitRequest.java (94%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/CommitGitRequest.java (95%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/EntitiesContentGitRequest.java (92%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/EntityContentGitRequest.java (95%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListBranchesGitRequest.java (87%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListEntitiesGitRequest.java (89%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/ListVersionsGitRequest.java (80%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/PendingGitRequest.java (96%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{ => data}/VoidGitRequest.java (85%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bafc93c19f..793ab0503c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -225,6 +226,18 @@ public class EntitiesVersionControlController extends BaseController { } } + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") + public DeferredResult compareEntityDataToVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid, + @RequestParam String versionId) throws ThingsboardException { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } @ApiOperation(value = "", notes = "" + "SINGLE_ENTITY:" + NEW_LINE + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 38d3b00213..777e84f348 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -24,18 +24,18 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 7f791163a0..73eaafe842 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -33,6 +34,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -42,6 +44,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -57,11 +60,13 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -75,6 +80,9 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static com.google.common.util.concurrent.Futures.transform; +import static com.google.common.util.concurrent.Futures.transformAsync; + @Service @TbCoreComponent @RequiredArgsConstructor @@ -109,7 +117,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); - return Futures.transformAsync(pendingCommit, commit -> { + return transformAsync(pendingCommit, commit -> { List> gitFutures = new ArrayList<>(); switch (request.getType()) { case SINGLE_ENTITY: { @@ -146,7 +154,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont break; } } - return Futures.transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + return transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); }, executor); } @@ -195,7 +203,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) + .findExistingByName(false) .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); @@ -298,6 +306,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } + @Override + public ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception { + HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); + + EntityId externalId = ((ExportableEntity) entity).getExternalId(); + if (externalId == null) externalId = entityId; + + EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(true) + .build()); + return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion), + JacksonUtil.toPrettyString(otherVersion)), rawDiff -> { + return new EntityDataDiff(currentVersion, otherVersion, rawDiff); + }, MoreExecutors.directExecutor()), MoreExecutors.directExecutor()); + } @Override public ListenableFuture> listBranches(TenantId tenantId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index f738582bb3..270dc5b38c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; @@ -24,6 +22,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -36,6 +35,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -54,12 +54,23 @@ import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ContentsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntityContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListBranchesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListEntitiesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListVersionsGitRequest; +import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; +import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -74,7 +85,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final DefaultEntitiesVersionControlService entitiesVersionControlService; private final Map> pendingRequestMap = new HashMap<>(); - private final ObjectMapper jsonMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, @@ -101,13 +111,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu SettableFuture future = SettableFuture.create(); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); - String entityDataJson; - try { - entityDataJson = jsonMapper.writeValueAsString(entityData); - } catch (IOException e) { - //TODO: analyze and return meaningful exceptions that we can show to the client; - throw new RuntimeException(e); - } + String entityDataJson = JacksonUtil.toPrettyString(entityData); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setAddMsg( @@ -176,10 +180,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); - - registerAndSend(request, builder -> builder.setListVersionRequest(requestMsg).build(), wrap(request.getFuture())); - - return request.getFuture(); + return sendRequest(request, builder -> builder.setListVersionRequest(requestMsg)); } @Override @@ -201,19 +202,32 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); - - registerAndSend(request, builder -> builder.setListEntitiesRequest(requestMsg).build(), wrap(request.getFuture())); - - return request.getFuture(); + return sendRequest(request, builder -> builder.setListEntitiesRequest(requestMsg)); } @Override public ListenableFuture> listBranches(TenantId tenantId) { ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); + } - registerAndSend(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()).build(), wrap(request.getFuture())); + @Override + public ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2) { + String path = entityType != null ? getRelativePath(entityType, externalId) : ""; + VersionsDiffGitRequest request = new VersionsDiffGitRequest(tenantId, path, versionId1, versionId2); + return sendRequest(request, builder -> builder.setVersionsDiffRequest(TransportProtos.VersionsDiffRequestMsg.newBuilder() + .setPath(request.getPath()) + .setVersionId1(request.getVersionId1()) + .setVersionId2(request.getVersionId2()) + .build())); + } - return request.getFuture(); + @Override + public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { + ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); + return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() + .setContent1(content1) + .setContent2(content2))); } @Override @@ -246,6 +260,14 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } } + private ListenableFuture sendRequest(PendingGitRequest request, Consumer enrichFunction) { + registerAndSend(request, builder -> { + enrichFunction.accept(builder); + return builder.build(); + }, wrap(request.getFuture())); + return request.getFuture(); + } + @Override @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { @@ -334,6 +356,23 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); ((EntitiesContentGitRequest) request).getFuture() .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + } else if (vcResponseMsg.hasVersionsDiffResponse()) { + TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); + List entityVersionsDiffList = diffResponse.getDiffList().stream() + .map(diff -> EntityVersionsDiff.builder() + .externalId(EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(diff.getEntityType()), + new UUID(diff.getEntityIdMSB(), diff.getEntityIdLSB()))) + .entityDataAtVersion1(StringUtils.isNotEmpty(diff.getEntityDataAtVersion1()) ? + toData(diff.getEntityDataAtVersion1()) : null) + .entityDataAtVersion2(StringUtils.isNotEmpty(diff.getEntityDataAtVersion2()) ? + toData(diff.getEntityDataAtVersion2()) : null) + .rawDiff(diff.getRawDiff()) + .build()) + .collect(Collectors.toList()); + ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); + } else if (vcResponseMsg.hasContentsDiffResponse()) { + String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); + ((ContentsDiffGitRequest) request).getFuture().set(diff); } } } @@ -354,7 +393,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @SuppressWarnings("rawtypes") @SneakyThrows private EntityExportData toData(String data) { - return jsonMapper.readValue(data, EntityExportData.class); + return JacksonUtil.fromString(data, EntityExportData.class); } private static TbQueueCallback wrap(SettableFuture future) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 1d8cfca00c..cb5ec52b6c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -17,11 +17,11 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadReques import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; -import java.util.concurrent.ExecutionException; public interface EntitiesVersionControlService { @@ -50,6 +49,8 @@ public interface EntitiesVersionControlService { ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; + ListenableFuture> listBranches(TenantId tenantId) throws Exception; EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index 00899a94e2..b7eb368055 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -29,6 +29,8 @@ import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import java.util.List; @@ -58,6 +60,10 @@ public interface GitVersionControlQueueService { ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); + + ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); + ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java index 86472e7882..09245eb895 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ClearRepositoryGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java index 3c83a34cfe..a989439d10 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/CommitGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.id.TenantId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java new file mode 100644 index 0000000000..7c3fb7a600 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class ContentsDiffGitRequest extends PendingGitRequest { + + private final String content1; + private final String content2; + + public ContentsDiffGitRequest(TenantId tenantId, String content1, String content2) { + super(tenantId); + this.content1 = content1; + this.content2 = content2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java index 2ded932e44..ad653edd0c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesContentGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java @@ -13,11 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java index c9ce3c76af..6a1d60a065 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntityContentGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import lombok.Getter; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java similarity index 87% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java index c8219641da..c045030dc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListBranchesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java index 0c7216ef49..3fc531735e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListEntitiesGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java similarity index 80% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java index 0c646b424c..d40a589646 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/ListVersionsGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java @@ -13,15 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.sync.vc.EntityVersion; -import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; - -import java.util.List; public class ListVersionsGitRequest extends PendingGitRequest> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java index e63ba04f45..fbec600ed0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/PendingGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import com.google.common.util.concurrent.SettableFuture; import lombok.Getter; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java new file mode 100644 index 0000000000..e73706609f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; + +import java.util.List; + +@Getter +public class VersionsDiffGitRequest extends PendingGitRequest> { + + private final String path; + private final String versionId1; + private final String versionId2; + + public VersionsDiffGitRequest(TenantId tenantId, String path, String versionId1, String versionId2) { + super(tenantId); + this.path = path; + this.versionId1 = versionId1; + this.versionId2 = versionId2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java index d829960ab4..c48bfa7a03 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VoidGitRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java @@ -13,12 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.data; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntityVersion; - -import java.util.List; public class VoidGitRequest extends PendingGitRequest { diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index a1cbddf6e8..5a61dfa46a 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -784,6 +784,34 @@ message EntitiesContentResponseMsg { repeated string data = 1; } +message VersionsDiffRequestMsg { + string path = 1; + string versionId1 = 2; + string versionId2 = 3; +} + +message VersionsDiffResponseMsg { + repeated EntityVersionsDiff diff = 1; +} + +message EntityVersionsDiff { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + string entityDataAtVersion1 = 4; + string entityDataAtVersion2 = 5; + string rawDiff = 6; +} + +message ContentsDiffRequestMsg { + string content1 = 1; + string content2 = 2; +} + +message ContentsDiffResponseMsg { + string diff = 1; +} + message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -804,6 +832,8 @@ message ToVersionControlServiceMsg { ListBranchesRequestMsg listBranchesRequest = 13; EntityContentRequestMsg entityContentRequest = 14; EntitiesContentRequestMsg entitiesContentRequest = 15; + VersionsDiffRequestMsg versionsDiffRequest = 16; + ContentsDiffRequestMsg contentsDiffRequest = 17; } message VersionControlResponseMsg { @@ -817,6 +847,8 @@ message VersionControlResponseMsg { ListVersionsResponseMsg listVersionsResponse = 8; EntityContentResponseMsg entityContentResponse = 9; EntitiesContentResponseMsg entitiesContentResponse = 10; + VersionsDiffResponseMsg versionsDiffResponse = 11; + ContentsDiffResponseMsg contentsDiffResponse = 12; } /** diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java new file mode 100644 index 0000000000..800583b0b8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +public class EntityDataDiff { + private EntityExportData currentVersion; + private EntityExportData otherVersion; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java new file mode 100644 index 0000000000..2104472047 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionsDiff { + private EntityId externalId; + private EntityExportData entityDataAtVersion1; + private EntityExportData entityDataAtVersion2; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 837986f82d..ffb1f25d3c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -23,5 +23,6 @@ import lombok.EqualsAndHashCode; public class EntityTypeVersionLoadConfig extends VersionLoadConfig { private boolean removeOtherEntities; + private boolean findExistingEntityByName; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 31b34dbcb7..8ed9c34694 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,6 +21,5 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; - private boolean findExistingEntityByName; } diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index f9df92966b..b6e2b54479 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -19,12 +19,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -32,6 +31,8 @@ import java.util.Set; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final ObjectMapper PRETTY_JSON_MAPPER = new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -96,6 +97,14 @@ public class JacksonUtil { } } + public static String toPrettyString(Object o) { + try { + return PRETTY_JSON_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public static JsonNode toJsonNode(String value) { if (value == null || value.isEmpty()) { return null; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 5fcb9985bd..47ab9fc2de 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -31,6 +31,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; @@ -91,6 +92,8 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; import java.util.stream.Collectors; +import static org.thingsboard.server.service.sync.vc.DefaultGitRepositoryService.fromRelativePath; + @Slf4j @TbVersionControlComponent @Service @@ -237,17 +240,18 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe vcService.fetch(ctx.getTenantId()); handleListBranches(ctx, msg.getListBranchesRequest()); } else if (msg.hasListEntitiesRequest()) { - vcService.fetch(ctx.getTenantId()); handleListEntities(ctx, msg.getListEntitiesRequest()); } else if (msg.hasListVersionRequest()) { vcService.fetch(ctx.getTenantId()); handleListVersions(ctx, msg.getListVersionRequest()); } else if (msg.hasEntityContentRequest()) { - vcService.fetch(ctx.getTenantId()); handleEntityContentRequest(ctx, msg.getEntityContentRequest()); } else if (msg.hasEntitiesContentRequest()) { - vcService.fetch(ctx.getTenantId()); handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } else if (msg.hasVersionsDiffRequest()) { + handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); + } else if (msg.hasContentsDiffRequest()) { + handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); } } } @@ -322,6 +326,31 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); } + private void handleVersionsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.VersionsDiffRequestMsg request) throws IOException { + List diffList = vcService.getVersionsDiffList(ctx.getTenantId(), request.getPath(), request.getVersionId1(), request.getVersionId2()).stream() + .map(diff -> { + EntityId entityId = fromRelativePath(diff.getFilePath()); + return TransportProtos.EntityVersionsDiff.newBuilder() + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEntityDataAtVersion1(diff.getFileContentAtCommit1()) + .setEntityDataAtVersion2(diff.getFileContentAtCommit2()) + .setRawDiff(diff.getDiffStringValue()) + .build(); + }) + .collect(Collectors.toList()); + + reply(ctx, builder -> builder.setVersionsDiffResponse(TransportProtos.VersionsDiffResponseMsg.newBuilder() + .addAllDiff(diffList))); + } + + private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { + String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); + reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() + .setDiff(diff))); + } + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); @@ -430,6 +459,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe reply(ctx, e, null); } + private void reply(VersionControlRequestCtx ctx, Function enrichFunction) { + reply(ctx, Optional.empty(), enrichFunction); + } + private void reply(VersionControlRequestCtx ctx, Optional e, Function enrichFunction) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); VersionControlResponseMsg.Builder builder = VersionControlResponseMsg.newBuilder() diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 45c5e563a6..a759ddbfa8 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import javax.annotation.PostConstruct; import java.io.File; @@ -167,6 +168,18 @@ public class DefaultGitRepositoryService implements GitRepositoryService { return repository.getFileContentAtCommit(relativePath, versionId); } + @Override + public List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getDiffList(versionId1, versionId2, path); + } + + @Override + public String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getContentsDiff(content1, content2); + } + @Override public List listBranches(TenantId tenantId) { GitRepository repository = checkRepository(tenantId); @@ -178,12 +191,6 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } } - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null, new PageLink(Integer.MAX_VALUE)).getData().stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - private GitRepository checkRepository(TenantId tenantId) { return Optional.ofNullable(repositories.get(tenantId)) .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); @@ -251,7 +258,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); } - private EntityId fromRelativePath(String path) { + public static EntityId fromRelativePath(String path) { EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); String entityId = StringUtils.substringBetween(path, "/", ".json"); return EntityIdFactory.getByTypeAndUuid(entityType, entityId); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 20ed03795d..921630a4ba 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -31,6 +31,12 @@ import org.eclipse.jgit.api.LsRemoteCommand; import org.eclipse.jgit.api.ResetCommand; import org.eclipse.jgit.api.TransportCommand; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.diff.EditList; +import org.eclipse.jgit.diff.HistogramDiff; +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.diff.RawTextComparator; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -45,6 +51,7 @@ import org.eclipse.jgit.transport.sshd.JGitKeyCache; import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; import org.eclipse.jgit.transport.sshd.SshdSessionFactory; import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; @@ -54,6 +61,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -207,7 +215,7 @@ public class GitRepository { RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { if (treeWalk == null) { - throw new IllegalArgumentException("Not found"); + throw new IllegalArgumentException("File not found"); } ObjectId blobId = treeWalk.getObjectId(0); try (ObjectReader objectReader = git.getRepository().newObjectReader()) { @@ -248,40 +256,65 @@ public class GitRepository { .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); } + public String getContentsDiff(String content1, String content2) throws IOException { + RawText rawContent1 = new RawText(content1.getBytes()); + RawText rawContent2 = new RawText(content2.getBytes()); -// public List getCommitChanges(Commit commit) throws IOException, GitAPIException { -// RevCommit revCommit = resolveCommit(commit.getId()); -// if (revCommit.getParentCount() == 0) { -// return null; // just takes the first parent of a commit, but should find a parent in branch provided -// } -// return execute(git.diff() -// .setOldTree(prepareTreeParser(git.getRepository().parseCommit(revCommit.getParent(0)))) -// .setNewTree(prepareTreeParser(revCommit))).stream() -// .map(diffEntry -> new Diff(diffEntry.getChangeType().name(), diffEntry.getOldPath(), diffEntry.getNewPath())) -// .collect(Collectors.toList()); -// } -// -// -// private AbstractTreeIterator prepareTreeParser(RevCommit revCommit) throws IOException { -// // from the commit we can build the tree which allows us to construct the TreeParser -// //noinspection Duplicates -// org.eclipse.jgit.lib.Repository repository = git.getRepository(); -// try (RevWalk walk = new RevWalk(repository)) { -// RevTree tree = walk.parseTree(revCommit.getTree().getId()); -// -// CanonicalTreeParser treeParser = new CanonicalTreeParser(); -// try (ObjectReader reader = repository.newObjectReader()) { -// treeParser.reset(reader, tree.getId()); -// } -// -// walk.dispose(); -// -// return treeParser; -// } -// } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + + EditList edits = new EditList(); + edits.addAll(new HistogramDiff().diff(RawTextComparator.DEFAULT, rawContent1, rawContent2)); + diffFormatter.format(edits, rawContent1, rawContent2); + return out.toString(); + } + + public List getDiffList(String commit1, String commit2, String path) throws IOException { + ObjectReader reader = git.getRepository().newObjectReader(); + + CanonicalTreeParser tree1Iter = new CanonicalTreeParser(); + ObjectId tree1 = resolveCommit(commit1).getTree(); + tree1Iter.reset(reader, tree1); + + CanonicalTreeParser tree2Iter = new CanonicalTreeParser(); + ObjectId tree2 = resolveCommit(commit2).getTree(); + tree2Iter.reset(reader, tree2); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + if (StringUtils.isNotEmpty(path)) { + diffFormatter.setPathFilter(PathFilter.create(path)); + } + + return diffFormatter.scan(tree1, tree2).stream() + .map(diffEntry -> { + Diff diff = new Diff(); + try { + out.reset(); + diffFormatter.format(diffEntry); + diff.setDiffStringValue(out.toString()); + diff.setFilePath(diffEntry.getChangeType() != DiffEntry.ChangeType.DELETE ? diffEntry.getNewPath() : diffEntry.getOldPath()); + diff.setChangeType(diffEntry.getChangeType()); + try { + diff.setFileContentAtCommit1(getFileContentAtCommit(diff.getFilePath(), commit1)); + } catch (IllegalArgumentException ignored) { + } + try { + diff.setFileContentAtCommit2(getFileContentAtCommit(diff.getFilePath(), commit2)); + } catch (IllegalArgumentException ignored) { + } + return diff; + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000L, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); } private RevCommit resolveCommit(String id) throws IOException { @@ -309,7 +342,7 @@ public class GitRepository { return null; }; - private static PageData iterableToPageData (Iterable iterable, + private static PageData iterableToPageData(Iterable iterable, Function mapper, PageLink pageLink, Function> comparatorFunction) { @@ -406,4 +439,13 @@ public class GitRepository { private final Set removed; } + @Data + public static class Diff { + private String filePath; + private DiffEntry.ChangeType changeType; + private String fileContentAtCommit1; + private String fileContentAtCommit2; + private String diffStringValue; + } + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index 9d57eee3ec..ec69e0c80e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import java.io.IOException; import java.util.List; @@ -60,5 +61,9 @@ public interface GitRepositoryService { String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException; + + String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException; + void fetch(TenantId tenantId) throws GitAPIException; } From 576c34b41ae13d150f18dcbe7703c32e3add083f Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 26 May 2022 15:28:45 +0300 Subject: [PATCH 076/156] Delete repository on Tenant Deletion --- .../server/service/entitiy/tenant/DefaultTbTenantService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 2a8fc1bf0a..22baed8366 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -27,8 +27,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.queue.TbQueueService; import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.Collections; +import java.util.concurrent.TimeUnit; @Service @TbCoreComponent @@ -38,6 +40,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T private final InstallScripts installScripts; private final TbQueueService tbQueueService; private final TenantProfileService tenantProfileService; + private final EntitiesVersionControlService versionControlService; @Override public Tenant save(Tenant tenant) throws ThingsboardException { @@ -70,6 +73,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T tenantService.deleteTenant(tenantId); tenantProfileCache.evict(tenantId); notificationEntityService.notifyDeleteTenant(tenant); + versionControlService.deleteVersionControlSettings(tenantId).get(1, TimeUnit.MINUTES); } catch (Exception e) { throw handleException(e); } From e86b37b4afe464fcd39475bd82d5d2a9943011ff Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 27 May 2022 13:14:18 +0300 Subject: [PATCH 077/156] UI: Initial implementation of entity versions diff component --- ui-ngx/angular.json | 6 +- ui-ngx/package.json | 2 + .../home/components/home-components.module.ts | 7 +- .../vc/entity-version-diff.component.html | 35 +++++++ .../vc/entity-version-diff.component.scss | 23 +++++ .../vc/entity-version-diff.component.ts | 92 +++++++++++++++++++ .../vc/entity-versions-table.component.html | 9 +- .../vc/entity-versions-table.component.ts | 23 +++++ .../src/app/shared/models/ace/ace.models.ts | 16 ++++ .../assets/locale/locale.constant-en_US.json | 4 +- ui-ngx/yarn.lock | 12 +++ 11 files changed, 223 insertions(+), 6 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts diff --git a/ui-ngx/angular.json b/ui-ngx/angular.json index f4d4412723..da4d21a84b 100644 --- a/ui-ngx/angular.json +++ b/ui-ngx/angular.json @@ -79,7 +79,8 @@ "node_modules/leaflet.markercluster/dist/MarkerCluster.Default.css", "node_modules/@geoman-io/leaflet-geoman-free/dist/leaflet-geoman.css", "node_modules/prismjs/themes/prism.css", - "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css" + "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css", + "node_modules/ace-diff/dist/ace-diff.min.css" ], "stylePreprocessorOptions": { "includePaths": [ @@ -130,7 +131,8 @@ "jstree", "qrcode", "wcwidth", - "leaflet-polylinedecorator" + "leaflet-polylinedecorator", + "ace-diff" ] }, "configurations": { diff --git a/ui-ngx/package.json b/ui-ngx/package.json index e54606dc08..fa21cc9525 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -41,6 +41,7 @@ "@ngx-translate/core": "^13.0.0", "@ngx-translate/http-loader": "^6.0.0", "ace-builds": "^1.4.13", + "ace-diff": "^3.0.3", "angular-gridster2": "~12.1.1", "angular2-hotkeys": "^2.4.0", "canvas-gauges": "^2.1.7", @@ -105,6 +106,7 @@ "@angular/compiler-cli": "^12.2.13", "@angular/language-service": "^12.2.13", "@ngtools/webpack": "~12.2.13", + "@types/ace-diff": "^2.1.1", "@types/canvas-gauges": "^2.1.4", "@types/flot": "^0.0.32", "@types/jasmine": "~3.10.2", diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 6134708176..9afe26d8f1 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -158,6 +158,7 @@ import { VersionControlComponent } from '@home/components/vc/version-control.com import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionExportComponent } from '@home/components/vc/entity-version-export.component'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; +import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; @NgModule({ declarations: @@ -286,7 +287,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio VersionControlComponent, EntityVersionsTableComponent, EntityVersionExportComponent, - EntityVersionRestoreComponent + EntityVersionRestoreComponent, + EntityVersionDiffComponent ], imports: [ CommonModule, @@ -409,7 +411,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio VersionControlComponent, EntityVersionsTableComponent, EntityVersionExportComponent, - EntityVersionRestoreComponent + EntityVersionRestoreComponent, + EntityVersionDiffComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html new file mode 100644 index 0000000000..f080fcaf99 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html @@ -0,0 +1,35 @@ + +
+ +

{{ 'version-control.diff-entity-with-version' | translate: {versionName} }}

+ + + + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss new file mode 100644 index 0000000000..5db7fd40e5 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .diff-viewer { + position: relative; + border: 1px solid #c0c0c0; + margin-top: 16px; + margin-bottom: 16px; + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts new file mode 100644 index 0000000000..f46375b152 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -0,0 +1,92 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, ElementRef, Input, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { getAceDiff } from '@shared/models/ace/ace.models'; + +@Component({ + selector: 'tb-entity-version-diff', + templateUrl: './entity-version-diff.component.html', + styleUrls: ['./entity-version-diff.component.scss'] +}) +export class EntityVersionDiffComponent extends PageComponent implements OnInit, OnDestroy { + + @ViewChild('diffViewer', {static: true}) + diffViewerElmRef: ElementRef; + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: () => void; + + differ: AceDiff; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + getAceDiff().subscribe((aceDiff) => { + this.differ = new aceDiff.default( + { + element: this.diffViewerElmRef.nativeElement, + left: { + copyLinkEnabled: false, + editable: false, + content: 'Left content!' + }, + right: { + copyLinkEnabled: false, + editable: false, + content: 'Right content!' + } + } as AceDiff.AceDiffConstructorOpts + ); + }); + } + + ngOnDestroy(): void { + if (this.differ) { + this.differ.destroy(); + } + } + + close(): void { + if (this.onClose) { + this.onClose(); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 2e5302881e..d931f2f9aa 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -100,10 +100,17 @@ - +
+ + + + {{ 'version-control.differences' | translate : {count: diffCount} }} + + + - - -
-
+
+
{{ 'version-control.current' | translate }}
+
+
{{ versionIdContent() }}
+
+ +
+
+ +
+ +
+
{{ resultMessage }}
+
+ +
+
+ diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss similarity index 96% rename from ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss rename to ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss index 46e55cf4d4..e8b3a5fbf6 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - .export-result-message { + .create-result-message { padding: 48px 8px 8px; text-align: center; } diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts new file mode 100644 index 0000000000..c152e6b447 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -0,0 +1,104 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + ComplexVersionCreateRequest, + createDefaultEntityTypesVersionCreate, + SyncStrategy, syncStrategyTranslationMap, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; + +@Component({ + selector: 'tb-complex-version-create', + templateUrl: './complex-version-create.component.html', + styleUrls: ['./complex-version-create.component.scss'] +}) +export class ComplexVersionCreateComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + onContentUpdated: () => void; + + createVersionFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + resultMessage: string; + + versionCreateResult: VersionCreationResult = null; + + versionCreateBranch: string = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + syncStrategy: [SyncStrategy.MERGE, Validators.required], + entityTypes: [createDefaultEntityTypesVersionCreate(), []], + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionCreateResult, this.versionCreateBranch); + } + } + + export(): void { + const request: ComplexVersionCreateRequest = { + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + syncStrategy: this.createVersionFormGroup.get('syncStrategy').value, + entityTypes: this.createVersionFormGroup.get('entityTypes').value, + type: VersionCreateRequestType.COMPLEX + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified && !result.removed) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + } else { + this.resultMessage = this.translate.instant('version-control.version-create-result', + {added: result.added, modified: result.modified, removed: result.removed}); + } + this.versionCreateResult = result; + this.versionCreateBranch = request.branch; + if (this.onContentUpdated) { + this.onContentUpdated(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html new file mode 100644 index 0000000000..74cfe9b891 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -0,0 +1,111 @@ + +
+
+ version-control.entity-types +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + version-control.sync-strategy + + + {{ 'version-control.default' | translate }} + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + + + {{ 'version-control.export-entity-relations' | translate }} + +
+
+
+ + {{ 'version-control.all-entities' | translate }} + + + +
+
+ + +
+
+
+ version-control.no-entity-types +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss new file mode 100644 index 0000000000..f90ad2a803 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-create { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts new file mode 100644 index 0000000000..bf2182dfcd --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -0,0 +1,247 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + SyncStrategy, + syncStrategyTranslationMap +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-entity-types-version-create', + templateUrl: './entity-types-version-create.component.html', + styleUrls: ['./entity-types-version-create.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + } + ] +}) +export class EntityTypesVersionCreateComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionCreateConfig}; + + private propagateChange = null; + + public entityTypesVersionCreateFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionCreateFormGroup = this.fb.group({ + entityTypes: [this.fb.array([]), [Validators.min(1)]] + }); + this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionCreateFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionCreateFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionCreateFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionCreateFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + syncStrategy: [config.syncStrategy === null ? 'default' : config.syncStrategy, []], + saveRelations: [config.saveRelations, []], + allEntities: [config.allEntities, []], + entityIds: [config.entityIds, [Validators.required]] + }) + } + ); + this.updateEntityTypeValidators(entityTypeControl); + entityTypeControl.get('config').get('allEntities').valueChanges.subscribe(() => { + this.updateEntityTypeValidators(entityTypeControl); + }); + return entityTypeControl; + } + + private updateEntityTypeValidators(entityTypeControl: AbstractControl): void { + const allEntities: boolean = entityTypeControl.get('config').get('allEntities').value; + if (allEntities) { + entityTypeControl.get('config').get('entityIds').disable({emitEvent: false}); + } else { + entityTypeControl.get('config').get('entityIds').enable({emitEvent: false}); + } + entityTypeControl.get('config').get('entityIds').updateValueAndValidity({emitEvent: false}); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionCreateConfig = { + syncStrategy: null, + saveRelations: false, + allEntities: true, + entityIds: [] + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: EntityTypeVersionCreateConfig = entityTypeControl.get('config').value; + let count = config?.entityIds?.length; + if (!isDefinedAndNotNull(count)) { + count = 0; + } + if (entityType) { + return this.translate.instant((config?.allEntities ? entityTypeTranslations.get(entityType).typePlural + : entityTypeTranslations.get(entityType).list), { count }); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionCreateConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + if ((modelValue[val.entityType].syncStrategy as any) === 'default') { + modelValue[val.entityType].syncStrategy = null; + } + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html similarity index 88% rename from ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html rename to ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 3838ea111d..5c575b93d4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-export.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -24,7 +24,7 @@ -
+
version-control.version-name - + {{ 'version-control.version-name-required' | translate }} @@ -54,13 +54,13 @@
-
{{ resultMessage }}
+
{{ resultMessage }}
+
+ formGroupName="config" style="min-height: 76px;"> {{ 'version-control.all-entities' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index ca0a8cbe85..8276e12d63 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -186,7 +186,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const createVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionCreateComponent, 'left', true, null, + this.viewContainerRef, EntityVersionCreateComponent, 'leftTop', true, null, { branch: this.branch, entityId: this.entityId, @@ -251,7 +251,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const diffVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionDiffComponent, 'left', true, null, + this.viewContainerRef, EntityVersionDiffComponent, 'leftTop', true, null, { branch: this.branch, versionName: entityVersion.name, @@ -275,7 +275,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.popoverService.hidePopover(trigger); } else { const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, { branch: this.branch, versionName: entityVersion.name, From 89c04a3460f1417d66594b714deb6f2ca1396c10 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 11:33:49 +0300 Subject: [PATCH 083/156] UI: Implement complex version load. --- .../home/components/home-components.module.ts | 10 +- .../vc/complex-version-load.component.html | 65 ++++++ .../vc/complex-version-load.component.scss | 27 +++ .../vc/complex-version-load.component.ts | 108 +++++++++ .../entity-types-version-create.component.ts | 2 +- .../entity-types-version-load.component.html | 90 ++++++++ .../entity-types-version-load.component.scss | 64 ++++++ .../vc/entity-types-version-load.component.ts | 210 ++++++++++++++++++ .../vc/entity-versions-table.component.html | 10 +- .../vc/entity-versions-table.component.ts | 22 ++ ui-ngx/src/app/shared/models/vc.models.ts | 37 ++- .../assets/locale/locale.constant-en_US.json | 10 +- 12 files changed, 640 insertions(+), 15 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f0dabc0309..e9fc3f8c9a 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -161,6 +161,8 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; +import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; @NgModule({ declarations: @@ -292,7 +294,9 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty EntityVersionRestoreComponent, EntityVersionDiffComponent, ComplexVersionCreateComponent, - EntityTypesVersionCreateComponent + EntityTypesVersionCreateComponent, + EntityTypesVersionLoadComponent, + ComplexVersionLoadComponent ], imports: [ CommonModule, @@ -418,7 +422,9 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty EntityVersionRestoreComponent, EntityVersionDiffComponent, ComplexVersionCreateComponent, - EntityTypesVersionCreateComponent + EntityTypesVersionCreateComponent, + EntityTypesVersionLoadComponent, + ComplexVersionLoadComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html new file mode 100644 index 0000000000..200dac7f04 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -0,0 +1,65 @@ + +
+
+ +

{{ 'version-control.restore-entities-from-version' | translate: {versionName} }}

+ + + + + +
+
+ + +
+
+ +
+ + +
+
+
+
+ {{ 'version-control.no-entities-restored' | translate }} +
+
{{ versionLoadResultMessage(versionLoadResult) }}
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss new file mode 100644 index 0000000000..eaeb773b2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.scss @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .load-result-message { + padding: 0 8px; + text-align: center; + &:first-child { + padding-top: 48px; + } + &:nth-last-child(2) { + padding-bottom: 8px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts new file mode 100644 index 0000000000..4b785fd384 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts @@ -0,0 +1,108 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + createDefaultEntityTypesVersionLoad, + EntityTypeVersionLoadRequest, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-complex-version-load', + templateUrl: './complex-version-load.component.html', + styleUrls: ['./complex-version-load.component.scss'] +}) +export class ComplexVersionLoadComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + onClose: (result: Array | null) => void; + + @Input() + onContentUpdated: () => void; + + loadVersionFormGroup: FormGroup; + + versionLoadResults: Array = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.loadVersionFormGroup = this.fb.group({ + entityTypes: [createDefaultEntityTypesVersionLoad(), []], + }); + } + + versionLoadResultMessage(result: VersionLoadResult): string { + const entityType = result.entityType; + let message = this.translate.instant(entityTypeTranslations.get(entityType).typePlural) + ': '; + const resultMessages: string[] = []; + if (result.created) { + resultMessages.push(this.translate.instant('version-control.created', {created: result.created})); + } + if (result.updated) { + resultMessages.push(this.translate.instant('version-control.updated', {updated: result.updated})); + } + if (result.deleted) { + resultMessages.push(this.translate.instant('version-control.deleted', {deleted: result.deleted})); + } + message += resultMessages.join(', ') + '.'; + return message; + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionLoadResults); + } + } + + restore(): void { + const request: EntityTypeVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + entityTypes: this.loadVersionFormGroup.get('entityTypes').value, + type: VersionLoadRequestType.ENTITY_TYPE + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + this.versionLoadResults = (result || []).filter(res => res.created || res.updated || res.deleted); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index bf2182dfcd..12fa811dc1 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -80,7 +80,7 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements ngOnInit(): void { this.entityTypesVersionCreateFormGroup = this.fb.group({ - entityTypes: [this.fb.array([]), [Validators.min(1)]] + entityTypes: [this.fb.array([]), []] }); this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { this.updateModel(); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html new file mode 100644 index 0000000000..77cb887d8a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -0,0 +1,90 @@ + +
+
+ version-control.entity-types +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.load-entities-relations' | translate }} + +
+
+
+ + +
+
+
+ version-control.no-entity-types +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss new file mode 100644 index 0000000000..b3e840198c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-load { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts new file mode 100644 index 0000000000..39709bebe1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -0,0 +1,210 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; + +@Component({ + selector: 'tb-entity-types-version-load', + templateUrl: './entity-types-version-load.component.html', + styleUrls: ['./entity-types-version-load.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + } + ] +}) +export class EntityTypesVersionLoadComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionLoadConfig}; + + private propagateChange = null; + + public entityTypesVersionLoadFormGroup: FormGroup; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionLoadFormGroup = this.fb.group({ + entityTypes: [this.fb.array([]), []] + }); + this.entityTypesVersionLoadFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionLoadFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionLoadFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionLoadFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionLoadFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionLoadConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + loadRelations: [config.loadRelations, []], + removeOtherEntities: [config.removeOtherEntities, []] + }) + } + ); + return entityTypeControl; + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionLoadConfig = { + loadRelations: false, + removeOtherEntities: false + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + if (entityType) { + return this.translate.instant(entityTypeTranslations.get(entityType).typePlural); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionLoadConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 54f3e69302..b4541c7182 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -107,7 +107,8 @@ - +
@@ -125,6 +126,13 @@ (click)="toggleRestoreEntityVersion($event, restoreVersionButton, entityVersion)"> restore +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 8276e12d63..be714e6f97 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -48,6 +48,7 @@ import { MatButton } from '@angular/material/button'; import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; @Component({ selector: 'tb-entity-versions-table', @@ -291,6 +292,27 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni } } + toggleRestoreEntitiesVersion($event: Event, restoreEntitiesVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreEntitiesVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreEntitiesVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionLoadComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + onClose: (result: Array | null) => { + restoreEntitiesVersionPopover.hide(); + } + }, {}, {}, {}, false); + } + } + versionIdContent(entityVersion: EntityVersion): string { let versionId = entityVersion.id; if (versionId.length > 7) { diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 9c5040f5f0..6e66daa24d 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -34,20 +34,11 @@ export interface VersionCreateConfig { saveRelations: boolean; } -export interface VersionLoadConfig { - loadRelations: boolean; -} - export enum VersionCreateRequestType { SINGLE_ENTITY = 'SINGLE_ENTITY', COMPLEX = 'COMPLEX' } -export enum VersionLoadRequestType { - SINGLE_ENTITY = 'SINGLE_ENTITY', - ENTITY_TYPE = 'ENTITY_TYPE' -} - export interface VersionCreateRequest { versionName: string; branch: string; @@ -97,6 +88,15 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: return res; } +export interface VersionLoadConfig { + loadRelations: boolean; +} + +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + export interface VersionLoadRequest { branch: string; versionId: string; @@ -109,6 +109,25 @@ export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { type: VersionLoadRequestType.SINGLE_ENTITY; } +export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { + removeOtherEntities: boolean; +} + +export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { + entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig}; + type: VersionLoadRequestType.ENTITY_TYPE; +} + +export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} { + const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + loadRelations: false, + removeOtherEntities: false + }; + } + return res; +} export interface BranchInfo { name: string; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 5dfece08ed..49b4f82c17 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3144,12 +3144,18 @@ "sync-strategy-overwrite": "Overwrite", "entity-types": "Entity types", "sync-strategy": "Sync strategy", - "default": "Default", "all-entities": "All entities", "no-entity-types": "No entity types configured", "add-entity-type": "Add entity type", "remove-all": "Remove all", - "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed." + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "load-entities-relations": "Load entities relations", + "remove-other-entities": "Remove other entities", + "restore-entities-from-version": "Restore entities from version '{{versionName}}'", + "no-entities-restored": "No entities restored", + "created": "{{created}} created", + "updated": "{{updated}} updated", + "deleted": "{{deleted}} deleted" }, "widget": { "widget-library": "Widgets Library", From bb9b58da5b13306bf32a2c0884e0d006f1b6e839 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 13:09:47 +0300 Subject: [PATCH 084/156] Add author to entity version --- .../DefaultEntitiesVersionControlService.java | 2 +- .../DefaultGitVersionControlQueueService.java | 35 +++++++++++++------ .../vc/GitVersionControlQueueService.java | 3 +- common/cluster-api/src/main/proto/queue.proto | 10 ++++-- .../common/data/sync/vc/EntityVersion.java | 1 + .../DefaultClusterVersionControlService.java | 6 ++-- .../sync/vc/DefaultGitRepositoryService.java | 12 +++++-- .../server/service/sync/vc/GitRepository.java | 7 ++-- .../server/service/sync/vc/PendingCommit.java | 8 ++++- .../vc/entity-versions-table.component.html | 8 ++++- .../vc/entity-versions-table.component.ts | 2 +- ui-ngx/src/app/shared/models/vc.models.ts | 1 + .../assets/locale/locale.constant-en_US.json | 1 + 13 files changed, 71 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 864a7365f6..0dc90fd109 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -120,7 +120,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings("UnstableApiUsage") @Override public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - var pendingCommit = gitServiceQueue.prepareCommit(user.getTenantId(), request); + var pendingCommit = gitServiceQueue.prepareCommit(user, request); return transformAsync(pendingCommit, commit -> { List> gitFutures = new ArrayList<>(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 9251fe392b..891c68b802 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -66,10 +67,7 @@ import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -96,12 +94,12 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request) { + public ListenableFuture prepareCommit(User user, VersionCreateRequest request) { SettableFuture future = SettableFuture.create(); - CommitGitRequest commit = new CommitGitRequest(tenantId, request); + CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); registerAndSend(commit, builder -> builder.setCommitRequest( - buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(request)).build() + buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() ).build(), wrap(future, commit)); return future; } @@ -356,7 +354,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName())); + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); @@ -405,7 +403,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { - return new EntityVersion(proto.getTs(), proto.getId(), proto.getName()); + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName(), proto.getAuthor()); } private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { @@ -453,8 +451,23 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return path; } - private static PrepareMsg getCommitPrepareMsg(VersionCreateRequest request) { - return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); + private static PrepareMsg getCommitPrepareMsg(User user, VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()) + .setBranchName(request.getBranch()).setAuthorName(getAuthorName(user)).setAuthorEmail(user.getEmail()).build(); + } + + private static String getAuthorName(User user) { + List parts = new ArrayList<>(); + if (StringUtils.isNotBlank(user.getFirstName())) { + parts.add(user.getFirstName()); + } + if (StringUtils.isNotBlank(user.getLastName())) { + parts.add(user.getLastName()); + } + if (parts.isEmpty()) { + parts.add(user.getName()); + } + return String.join(" ", parts); } private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, EntitiesVersionControlSettings settings) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index b7eb368055..5a4ffe1706 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -36,7 +37,7 @@ import java.util.List; public interface GitVersionControlQueueService { - ListenableFuture prepareCommit(TenantId tenantId, VersionCreateRequest request); + ListenableFuture prepareCommit(User user, VersionCreateRequest request); ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 969ee0261c..3079978bce 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -692,14 +692,17 @@ message CommitResponseMsg { int64 ts = 1; string commitId = 2; string name = 3; - int32 added = 4; - int32 modified = 5; - int32 removed = 6; + string author = 4; + int32 added = 5; + int32 modified = 6; + int32 removed = 7; } message PrepareMsg { string commitMsg = 1; string branchName = 2; + string authorName = 3; + string authorEmail = 4; } message AddMsg { @@ -733,6 +736,7 @@ message EntityVersionProto { int64 ts = 1; string id = 2; string name = 3; + string author = 4; } message ListVersionsResponseMsg { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 50d3fef8a8..c90336d3d7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -26,4 +26,5 @@ public class EntityVersion { private long timestamp; private String id; private String name; + private String author; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index cd112324f6..dc42a61257 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -312,7 +312,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTotalElements(data.getTotalElements()) .setHasNext(data.hasNext()) .addAllVersions(data.getData().stream().map( - v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).build() + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).setAuthor(v.getAuthor()).build() ).collect(Collectors.toList()))) ); } @@ -397,7 +397,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private void prepareCommit(VersionControlRequestCtx ctx, UUID txId, PrepareMsg prepareMsg) { var tenantId = ctx.getTenantId(); - var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), prepareMsg.getCommitMsg()); + var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), + prepareMsg.getCommitMsg(), prepareMsg.getAuthorName(), prepareMsg.getAuthorEmail()); PendingCommit old = pendingCommitMap.get(tenantId); if (old != null) { doAbortCurrentCommit(tenantId, old); @@ -460,6 +461,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .setTs(result.getVersion().getTimestamp()) .setCommitId(result.getVersion().getId()) .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()) .setAdded(result.getAdded()) .setModified(result.getModified()) .setRemoved(result.getRemoved()))); diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index a759ddbfa8..8c30d3097a 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -117,7 +117,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setModified(status.getModified().size()); result.setRemoved(status.getRemoved().size()); - GitRepository.Commit gitCommit = repository.commit(commit.getVersionName()); + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); repository.push(commit.getWorkingBranch(), commit.getBranch()); result.setVersion(toVersion(gitCommit)); @@ -255,7 +255,15 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage()); + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage(), this.getAuthor(commit)); + } + + private String getAuthor(GitRepository.Commit commit) { + String author = String.format("<%s>", commit.getAuthorEmail()); + if (StringUtils.isNotBlank(commit.getAuthorName())) { + author = String.format("%s %s", commit.getAuthorName(), author); + } + return author; } public static EntityId fromRelativePath(String path) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5945067284..938094cfb8 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -255,8 +255,9 @@ public class GitRepository { return new Status(status.getAdded(), modified, status.getRemoved()); } - public Commit commit(String message) throws GitAPIException { + public Commit commit(String message, String authorName, String authorEmail) throws GitAPIException { RevCommit revCommit = execute(git.commit() + .setAuthor(authorName, authorEmail) .setMessage(message)); return toCommit(revCommit); } @@ -325,7 +326,8 @@ public class GitRepository { } private Commit toCommit(RevCommit revCommit) { - return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), revCommit.getFullMessage(), revCommit.getAuthorIdent().getName()); + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), + revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); } private RevCommit resolveCommit(String id) throws IOException { @@ -470,6 +472,7 @@ public class GitRepository { private final String id; private final String message; private final String authorName; + private final String authorEmail; } @Data diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index b2eccd79db..ccd5fc685e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -30,12 +30,18 @@ public class PendingCommit { private String branch; private String versionName; - public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName) { + private String authorName; + + private String authorEmail; + + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { this.tenantId = tenantId; this.nodeId = nodeId; this.txId = txId; this.branch = branch; this.versionName = versionName; + this.authorName = authorName; + this.authorEmail = authorEmail; this.workingBranch = txId.toString(); } } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b4541c7182..d2dbc119bf 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -101,11 +101,17 @@ - {{ 'version-control.version-name' | translate }} + {{ 'version-control.version-name' | translate }} {{ entityVersion.name }} + + {{ 'version-control.author' | translate }} + + {{ entityVersion.author }} + + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index be714e6f97..589800e143 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -62,7 +62,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni @Input() singleEntityMode = false; - displayedColumns = ['timestamp', 'id', 'name', 'actions']; + displayedColumns = ['timestamp', 'id', 'name', 'author', 'actions']; pageLink: PageLink; textSearchMode = false; dataSource: EntityVersionsDatasource; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 6e66daa24d..0ad52d2e0b 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -138,6 +138,7 @@ export interface EntityVersion { timestamp: number; id: string; name: string; + author: string; } export interface VersionCreationResult { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 49b4f82c17..4a4e228cc2 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3119,6 +3119,7 @@ "create-entity-version": "Create entity version", "version-name": "Version name", "version-name-required": "Version name is required", + "author": "Author", "export-entity-relations": "Export entity relations", "entity-versions": "Entity versions", "versions": "Versions", From 07b6813465f958db07a9945f347a2f59b45af1d5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 13:19:16 +0300 Subject: [PATCH 085/156] Separate AutoCommit settings --- .../server/controller/AdminController.java | 72 +++++++++++-- .../DefaultEntitiesVersionControlService.java | 62 ++++++----- .../DefaultGitVersionControlQueueService.java | 12 +-- ...efaultTbVersionControlSettingsService.java | 102 ------------------ .../vc/EntitiesVersionControlService.java | 9 +- .../vc/GitVersionControlQueueService.java | 6 +- ...AbstractVersionControlSettingsService.java | 80 ++++++++++++++ .../AutoCommitSettingsCaffeineCache.java} | 14 ++- .../AutoCommitSettingsRedisCache.java} | 17 ++- .../DefaultTbAutoCommitSettingsService.java | 36 +++++++ .../TbAutoCommitSettingsService.java | 30 ++++++ .../DefaultTbRepositorySettingsService.java | 63 +++++++++++ .../RepositorySettingsCaffeineCache.java | 34 ++++++ .../RepositorySettingsRedisCache.java | 36 +++++++ .../TbRepositorySettingsService.java} | 12 +-- .../src/main/resources/thingsboard.yml | 9 +- .../server/common/data/CacheConstants.java | 3 +- .../data/sync/vc/AutoCommitSettings.java | 27 +++++ ...lSettings.java => RepositorySettings.java} | 11 +- .../DefaultClusterVersionControlService.java | 6 +- .../sync/vc/DefaultGitRepositoryService.java | 8 +- .../server/service/sync/vc/GitRepository.java | 12 +-- .../service/sync/vc/GitRepositoryService.java | 8 +- .../sync/vc/VersionControlRequestCtx.java | 6 +- 24 files changed, 467 insertions(+), 208 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{VersionControlSettingsCaffeineCache.java => autocommit/AutoCommitSettingsCaffeineCache.java} (66%) rename application/src/main/java/org/thingsboard/server/service/sync/vc/{VersionControlSettingsRedisCache.java => autocommit/AutoCommitSettingsRedisCache.java} (58%) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java rename application/src/main/java/org/thingsboard/server/service/sync/vc/{TbVersionControlSettingsService.java => repository/TbRepositorySettingsService.java} (60%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java rename common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/{EntitiesVersionControlSettings.java => RepositorySettings.java} (80%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index cbc6ff2bca..67665ad8c7 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -29,24 +29,23 @@ import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.UpdateMessage; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.update.UpdateService; import static org.thingsboard.server.controller.ControllerConstants.*; -import static org.thingsboard.server.controller.ControllerConstants.DEVICE_ID; @RestController @TbCoreComponent @@ -68,6 +67,9 @@ public class AdminController extends BaseController { @Autowired private EntitiesVersionControlService versionControlService; + @Autowired + private TbAutoCommitSettingsService autoCommitSettingsService; + @Autowired private UpdateService updateService; @@ -194,10 +196,10 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('TENANT_ADMIN')") @GetMapping("/vcSettings") @ResponseBody - public EntitiesVersionControlSettings getVersionControlSettings() throws ThingsboardException { + public RepositorySettings getVersionControlSettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - EntitiesVersionControlSettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); versionControlSettings.setPassword(null); versionControlSettings.setPrivateKey(null); versionControlSettings.setPrivateKeyPassword(null); @@ -225,9 +227,9 @@ public class AdminController extends BaseController { notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vcSettings") - public DeferredResult saveVersionControlSettings(@RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + public DeferredResult saveVersionControlSettings(@RequestBody RepositorySettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); + ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); return wrapFuture(Futures.transform(future, savedSettings -> { savedSettings.setPassword(null); savedSettings.setPrivateKey(null); @@ -251,13 +253,65 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", + notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vc/autoCommitSettings") + @ResponseBody + public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return checkNotNull(autoCommitSettingsService.get(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", + notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/vc/autoCommitSettings/exists") + @ResponseBody + public Boolean autoCommitSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return autoCommitSettingsService.get(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", + notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/vc/autoCommitSettings") + public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + return autoCommitSettingsService.save(getTenantId(), settings); + } + + @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", + notes = "Deletes the version control settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); + autoCommitSettingsService.delete(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check version control access (checkVersionControlAccess)", notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) public DeferredResult checkVersionControlAccess( @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") - @RequestBody EntitiesVersionControlSettings settings) throws ThingsboardException { + @RequestBody RepositorySettings settings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); settings = checkNotNull(settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 864a7365f6..3060b05b1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -27,7 +27,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; -import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; @@ -44,7 +43,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -62,19 +61,18 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; -import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.time.Instant; -import java.time.ZonedDateTime; -import java.time.temporal.TemporalUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -94,7 +92,8 @@ import static com.google.common.util.concurrent.Futures.transformAsync; @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - private final TbVersionControlSettingsService vcSettingsService; + private final TbRepositorySettingsService repositorySettingsService; + private final TbAutoCommitSettingsService autoCommitSettingsService; private final GitVersionControlQueueService gitServiceQueue; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; @@ -336,16 +335,16 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId) { - return vcSettingsService.get(tenantId); + public RepositorySettings getVersionControlSettings(TenantId tenantId) { + return repositorySettingsService.get(tenantId); } @Override - public ListenableFuture saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - var restoredSettings = this.vcSettingsService.restore(tenantId, versionControlSettings); + public ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings) { + var restoredSettings = this.repositorySettingsService.restore(tenantId, versionControlSettings); try { var future = gitServiceQueue.initRepository(tenantId, restoredSettings); - return Futures.transform(future, f -> vcSettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); + return Futures.transform(future, f -> repositorySettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); } catch (Exception e) { log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e); throw new RuntimeException("Failed to init repository!", e); @@ -354,7 +353,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception { - if (vcSettingsService.delete(tenantId)) { + if (repositorySettingsService.delete(tenantId)) { return gitServiceQueue.clearRepository(tenantId); } else { return Futures.immediateFuture(null); @@ -362,8 +361,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } @Override - public ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { - settings = this.vcSettingsService.restore(tenantId, settings); + public ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws ThingsboardException { + settings = this.repositorySettingsService.restore(tenantId, settings); try { return gitServiceQueue.testRepository(tenantId, settings); } catch (Exception e) { @@ -374,22 +373,29 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { - var settings = vcSettingsService.get(user.getTenantId()); + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } var entityType = entityId.getEntityType(); - if (settings != null && settings.getAutoCommitSettings() != null && settings.getAutoCommitSettings().containsKey(entityType)) { - AutoVersionCreateConfig autoCommitConfig = settings.getAutoCommitSettings().get(entityType); - SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); - var autoCommitBranchName = autoCommitConfig.getBranch(); - if (StringUtils.isEmpty(autoCommitBranchName)) { - autoCommitBranchName = StringUtils.isNotEmpty(settings.getDefaultBranch()) ? settings.getDefaultBranch() : "auto-commits"; - } - vcr.setBranch(autoCommitBranchName); - vcr.setVersionName("auto-commit by " + user.getEmail() + " at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); - vcr.setEntityId(entityId); - vcr.setConfig(autoCommitConfig); - return saveEntitiesVersion(user, vcr); + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; } - return Futures.immediateFuture(null); + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setEntityId(entityId); + vcr.setConfig(autoCommitConfig); + return saveEntitiesVersion(user, vcr); } private String getCauseMessage(Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 9251fe392b..7f501522af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; @@ -271,7 +271,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } private void registerAndSend(PendingGitRequest request, - Function enrichFunction, EntitiesVersionControlSettings settings, TbQueueCallback callback) { + Function enrichFunction, RepositorySettings settings, TbQueueCallback callback) { if (!request.getFuture().isDone()) { pendingRequestMap.putIfAbsent(request.getRequestId(), request); var requestBody = enrichFunction.apply(newRequestProto(request, settings)); @@ -307,7 +307,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + public ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings) { VoidGitRequest request = new VoidGitRequest(tenantId); registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() @@ -317,7 +317,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } @Override - public ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + public ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings) { VoidGitRequest request = new VoidGitRequest(tenantId); registerAndSend(request, builder -> builder @@ -457,7 +457,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()).setBranchName(request.getBranch()).build(); } - private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, EntitiesVersionControlSettings settings) { + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, RepositorySettings settings) { var tenantId = request.getTenantId(); var requestId = request.getRequestId(); var builder = ToVersionControlServiceMsg.newBuilder() @@ -466,7 +466,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) .setRequestIdMSB(requestId.getMostSignificantBits()) .setRequestIdLSB(requestId.getLeastSignificantBits()); - EntitiesVersionControlSettings vcSettings = settings; + RepositorySettings vcSettings = settings; if (vcSettings == null && request.requiresSettings()) { vcSettings = entitiesVersionControlService.getVersionControlSettings(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java deleted file mode 100644 index 1d1fff81a0..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultTbVersionControlSettingsService.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vc; - -import lombok.RequiredArgsConstructor; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.queue.util.TbCoreComponent; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -public class DefaultTbVersionControlSettingsService implements TbVersionControlSettingsService { - - public static final String SETTINGS_KEY = "entitiesVersionControl"; - private final AdminSettingsService adminSettingsService; - private final TbTransactionalCache cache; - - @Override - public EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings settings) { - EntitiesVersionControlSettings storedSettings = get(tenantId); - if (storedSettings != null) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { - settings.setPassword(storedSettings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { - settings.setPrivateKey(storedSettings.getPrivateKey()); - if (settings.getPrivateKeyPassword() == null) { - settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); - } - } - } - return settings; - } - - @Override - public EntitiesVersionControlSettings get(TenantId tenantId) { - EntitiesVersionControlSettings settings = cache.getAndPutInTransaction(tenantId, () -> { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings != null) { - try { - return JacksonUtil.convertValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - } - return null; - }, true); - if (settings != null) { - settings = new EntitiesVersionControlSettings(settings); - } - return settings; - } - - @Override - public EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings) { - AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, SETTINGS_KEY); - if (adminSettings == null) { - adminSettings = new AdminSettings(); - adminSettings.setKey(SETTINGS_KEY); - adminSettings.setTenantId(tenantId); - } - adminSettings.setJsonValue(JacksonUtil.valueToTree(versionControlSettings)); - AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); - EntitiesVersionControlSettings savedVersionControlSettings; - try { - savedVersionControlSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), EntitiesVersionControlSettings.class); - } catch (Exception e) { - throw new RuntimeException("Failed to load version control settings!", e); - } - //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. - cache.evict(tenantId); - return savedVersionControlSettings; - } - - @Override - public boolean delete(TenantId tenantId) { - boolean result = adminSettingsService.deleteAdminSettings(tenantId, SETTINGS_KEY); - cache.evict(tenantId); - return result; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 836a75e021..447ae1d021 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.sync.vc; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -24,7 +23,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -54,13 +53,13 @@ public interface EntitiesVersionControlService { ListenableFuture> listBranches(TenantId tenantId) throws Exception; - EntitiesVersionControlSettings getVersionControlSettings(TenantId tenantId); + RepositorySettings getVersionControlSettings(TenantId tenantId); - ListenableFuture saveVersionControlSettings(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings); ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception; - ListenableFuture checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index b7eb368055..221206f5bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -64,9 +64,9 @@ public interface GitVersionControlQueueService { ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); - ListenableFuture initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); - ListenableFuture testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); ListenableFuture clearRepository(TenantId tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java new file mode 100644 index 0000000000..e826c32c56 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import java.io.Serializable; + +public abstract class TbAbstractVersionControlSettingsService { + + private final String settingsKey; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + private final Class clazz; + + public TbAbstractVersionControlSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache, Class clazz, String settingsKey) { + this.adminSettingsService = adminSettingsService; + this.cache = cache; + this.clazz = clazz; + this.settingsKey = settingsKey; + } + + public T get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, settingsKey); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load " + settingsKey + " settings!", e); + } + } + return null; + }, true); + } + + public T save(TenantId tenantId, T settings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, settingsKey); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(settingsKey); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + T savedSettings; + try { + savedSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load auto commit settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedSettings; + } + + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettings(tenantId, settingsKey); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java similarity index 66% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java index 8cab03ca29..47b9e5f5d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsCaffeineCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java @@ -13,24 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.autocommit; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CaffeineTbTransactionalCache; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) -@Service("VersionControlCache") -public class VersionControlSettingsCaffeineCache extends CaffeineTbTransactionalCache { +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsCaffeineCache extends CaffeineTbTransactionalCache { - public VersionControlSettingsCaffeineCache(CacheManager cacheManager) { - super(cacheManager, CacheConstants.VC_SETTINGS_CACHE); + public AutoCommitSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.AUTO_COMMIT_SETTINGS_CACHE); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java index 9c778d6e4f..f88b1cf6bd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlSettingsRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java @@ -13,29 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.autocommit; -import com.google.protobuf.InvalidProtocolBufferException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.serializer.RedisSerializer; -import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.cache.TbRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") -@Service("VersionControlCache") -public class VersionControlSettingsRedisCache extends RedisTbTransactionalCache { +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache { - public VersionControlSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.VC_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java new file mode 100644 index 0000000000..b6e8d45ec2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbAutoCommitSettingsService extends TbAbstractVersionControlSettingsService implements TbAutoCommitSettingsService { + + public static final String SETTINGS_KEY = "autoCommitSettings"; + + public DefaultTbAutoCommitSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, AutoCommitSettings.class, SETTINGS_KEY); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java new file mode 100644 index 0000000000..51978481e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbAutoCommitSettingsService { + + AutoCommitSettings get(TenantId tenantId); + + AutoCommitSettings save(TenantId tenantId, AutoCommitSettings settings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java new file mode 100644 index 0000000000..608efec870 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbRepositorySettingsService extends TbAbstractVersionControlSettingsService implements TbRepositorySettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + + public DefaultTbRepositorySettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, RepositorySettings.class, SETTINGS_KEY); + } + + @Override + public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { + RepositorySettings storedSettings = get(tenantId); + if (storedSettings != null) { + VersionControlAuthMethod authMethod = settings.getAuthMethod(); + if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public RepositorySettings get(TenantId tenantId) { + RepositorySettings settings = super.get(tenantId); + if (settings != null) { + settings = new RepositorySettings(settings); + } + return settings; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java new file mode 100644 index 0000000000..60b7f50e12 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("RepositorySettingsCache") +public class RepositorySettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public RepositorySettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.REPOSITORY_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java new file mode 100644 index 0000000000..3cccb6d24e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("RepositorySettingsCache") +public class RepositorySettingsRedisCache extends RedisTbTransactionalCache { + + public RepositorySettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java similarity index 60% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java index 178499c6c7..946d06c87c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbVersionControlSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc; +package org.thingsboard.server.service.sync.vc.repository; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -public interface TbVersionControlSettingsService { +public interface TbRepositorySettingsService { - EntitiesVersionControlSettings restore(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + RepositorySettings restore(TenantId tenantId, RepositorySettings versionControlSettings); - EntitiesVersionControlSettings get(TenantId tenantId); + RepositorySettings get(TenantId tenantId); - EntitiesVersionControlSettings save(TenantId tenantId, EntitiesVersionControlSettings versionControlSettings); + RepositorySettings save(TenantId tenantId, RepositorySettings versionControlSettings); boolean delete(TenantId tenantId); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 31e5d7a76e..9663396d7e 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -433,9 +433,12 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" - vcSettings: - timeToLiveInMinutes: "${CACHE_SPECS_VC_SETTINGS_TTL:1440}" - maxSize: "${CACHE_SPECS_VC_SETTINGS_MAX_SIZE:10000}" + repositorySettings: + timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" + autoCommitSettings: + timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" redis: # standalone or cluster diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 85ea116f0a..94ca9db5a5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -31,5 +31,6 @@ public class CacheConstants { public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; - public static final String VC_SETTINGS_CACHE = "vcSettings"; + public static final String REPOSITORY_SETTINGS_CACHE = "repositorySettings"; + public static final String AUTO_COMMIT_SETTINGS_CACHE = "autoCommitSettings"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java new file mode 100644 index 0000000000..2120ec4c44 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; + +import java.util.HashMap; + +public class AutoCommitSettings extends HashMap { + + private static final long serialVersionUID = -5757067601838792059L; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java similarity index 80% rename from common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index 7764274d22..c576f82298 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.vc; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; @@ -24,7 +25,8 @@ import java.util.HashMap; import java.util.Map; @Data -public class EntitiesVersionControlSettings implements Serializable { +@JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. +public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; private String repositoryUri; @@ -36,12 +38,10 @@ public class EntitiesVersionControlSettings implements Serializable { private String privateKeyPassword; private String defaultBranch; - private Map autoCommitSettings; - - public EntitiesVersionControlSettings() { + public RepositorySettings() { } - public EntitiesVersionControlSettings(EntitiesVersionControlSettings settings) { + public RepositorySettings(RepositorySettings settings) { this.repositoryUri = settings.getRepositoryUri(); this.authMethod = settings.getAuthMethod(); this.username = settings.getUsername(); @@ -50,6 +50,5 @@ public class EntitiesVersionControlSettings implements Serializable { this.privateKey = settings.getPrivateKey(); this.privateKeyPassword = settings.getPrivateKeyPassword(); this.defaultBranch = settings.getDefaultBranch(); - this.autoCommitSettings = settings.getAutoCommitSettings() != null ? new HashMap<>(settings.getAutoCommitSettings()) : new HashMap<>(); } } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index cd112324f6..9463608b76 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -495,8 +495,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); } - private EntitiesVersionControlSettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { - Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + private RepositorySettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); if (settingsOpt.isPresent()) { return settingsOpt.get(); } else { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index a759ddbfa8..b12e03e762 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -216,13 +216,13 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + public void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception { Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); GitRepository.test(settings, repositoryDirectory.toFile()); } @Override - public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + public void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception { clearRepository(tenantId); log.debug("[{}] Init tenant repository started.", tenantId); Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); @@ -238,7 +238,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService { } @Override - public EntitiesVersionControlSettings getRepositorySettings(TenantId tenantId) throws Exception { + public RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception { var gitRepository = repositories.get(tenantId); return gitRepository != null ? gitRepository.getSettings() : null; } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 5945067284..b5bc6355d4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -58,7 +58,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; @@ -77,14 +77,14 @@ public class GitRepository { private final Git git; @Getter - private final EntitiesVersionControlSettings settings; + private final RepositorySettings settings; private final CredentialsProvider credentialsProvider; private final SshdSessionFactory sshSessionFactory; @Getter private final String directory; - private GitRepository(Git git, EntitiesVersionControlSettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { this.git = git; this.settings = settings; this.credentialsProvider = credentialsProvider; @@ -92,7 +92,7 @@ public class GitRepository { this.directory = directory; } - public static GitRepository clone(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { @@ -109,7 +109,7 @@ public class GitRepository { return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static GitRepository open(File directory, EntitiesVersionControlSettings settings) throws IOException { + public static GitRepository open(File directory, RepositorySettings settings) throws IOException { Git git = Git.open(directory); CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; @@ -121,7 +121,7 @@ public class GitRepository { return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); } - public static void test(EntitiesVersionControlSettings settings, File directory) throws GitAPIException { + public static void test(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java index ec69e0c80e..057fa9c1ab 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -19,7 +19,7 @@ import org.eclipse.jgit.api.errors.GitAPIException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; @@ -39,11 +39,11 @@ public interface GitRepositoryService { List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; - void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception; - void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception; - EntitiesVersionControlSettings getRepositorySettings(TenantId tenantId) throws Exception; + RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception; void clearRepository(TenantId tenantId) throws IOException; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java index 6541a2978a..26ab0d94c1 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.sync.vc; import lombok.Data; import lombok.RequiredArgsConstructor; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import java.util.UUID; @@ -29,9 +29,9 @@ public class VersionControlRequestCtx { private final String nodeId; private final UUID requestId; private final TenantId tenantId; - private final EntitiesVersionControlSettings settings; + private final RepositorySettings settings; - public VersionControlRequestCtx(ToVersionControlServiceMsg msg, EntitiesVersionControlSettings settings) { + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, RepositorySettings settings) { this.nodeId = msg.getNodeId(); this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); From 32f2f4ccd07c044b9396c42b556146c4daa3c832 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 13:19:36 +0300 Subject: [PATCH 086/156] Export/import of attributes --- .../impl/DefaultEntityExportService.java | 79 ++++++++++++--- .../impl/BaseEntityImportService.java | 97 +++++++++++++++---- .../DefaultEntitiesVersionControlService.java | 15 ++- .../data/sync/ie/AttributeExportData.java | 30 ++++++ .../common/data/sync/ie/EntityExportData.java | 2 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 9 files changed, 192 insertions(+), 35 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 31bf768e90..8e723f8793 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -19,22 +19,31 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; @Service @TbCoreComponent @@ -45,6 +54,8 @@ public class DefaultEntityExportService relations = new ArrayList<>(); + List relations = exportRelations(user, entity); + exportData.setRelations(relations); + } + if (exportSettings.isExportAttributes()) { + Map> attributes = exportAttributes(user, entity); + exportData.setAttributes(attributes); + } + } - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } - relations.addAll(inboundRelations); + private List exportRelations(SecurityUser user, E entity) throws ThingsboardException { + List relations = new ArrayList<>(); - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } - relations.addAll(outboundRelations); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + relations.addAll(inboundRelations); - exportData.setRelations(relations); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } + relations.addAll(outboundRelations); + return relations; + } + + private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ_ATTRIBUTES); + + List scopes; + if (entity.getId().getEntityType() == EntityType.DEVICE) { + scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + } else { + scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + } + Map> attributes = new HashMap<>(); + scopes.forEach(scope -> { + try { + attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream() + .map(attribute -> { + AttributeExportData attributeExportData = new AttributeExportData(); + attributeExportData.setKey(attribute.getKey()); + attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs()); + attributeExportData.setStrValue(attribute.getStrValue().orElse(null)); + attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null)); + attributeExportData.setLongValue(attribute.getLongValue().orElse(null)); + attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null)); + attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null)); + return attributeExportData; + }) + .collect(Collectors.toList())); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + return attributes; } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 203b0caa16..caa1e20ad7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import com.google.common.util.concurrent.FutureCallback; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -29,23 +32,36 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; +@Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy @@ -53,6 +69,8 @@ public abstract class BaseEntityImportService { - if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) { - return; - } - - List relations = new ArrayList<>(exportData.getRelations()); + if (importSettings.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(user, exportData.getRelations(), importResult); + } + if (importSettings.isSaveAttributes() && exportData.getAttributes() != null) { + importAttributes(user, exportData.getAttributes(), importResult); + } + } + private void importRelations(SecurityUser user, List relations, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { for (EntityRelation relation : relations) { - if (!relation.getTo().equals(savedEntity.getId())) { + if (!relation.getTo().equals(entity.getId())) { HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); relation.setTo(to.getId()); } - if (!relation.getFrom().equals(savedEntity.getId())) { + if (!relation.getFrom().equals(entity.getId())) { HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); } } - if (oldEntity != null) { + if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { if (!relations.contains(existingRelation)) { EntityId otherEntity = null; - if (!existingRelation.getTo().equals(savedEntity.getId())) { + if (!existingRelation.getTo().equals(entity.getId())) { otherEntity = existingRelation.getTo(); - } else if (!existingRelation.getFrom().equals(savedEntity.getId())) { + } else if (!existingRelation.getFrom().equals(entity.getId())) { otherEntity = existingRelation.getFrom(); } if (otherEntity != null) { @@ -161,6 +183,44 @@ public abstract class BaseEntityImportService> attributes, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + attributes.forEach((scope, attributesExportData) -> { + List attributeKvEntries = attributesExportData.stream() + .map(attributeExportData -> { + KvEntry kvEntry; + String key = attributeExportData.getKey(); + if (attributeExportData.getStrValue() != null) { + kvEntry = new StringDataEntry(key, attributeExportData.getStrValue()); + } else if (attributeExportData.getBooleanValue() != null) { + kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue()); + } else if (attributeExportData.getDoubleValue() != null) { + kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue()); + } else if (attributeExportData.getLongValue() != null) { + kvEntry = new LongDataEntry(key, attributeExportData.getLongValue()); + } else if (attributeExportData.getJsonValue() != null) { + kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue()); + } else { + throw new IllegalArgumentException("Invalid attribute export data"); + } + return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs()); + }) + .collect(Collectors.toList()); + // fixme: attributes are saved outside the transaction + tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void unused) {} + + @Override + public void onFailure(Throwable thr) { + log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr); + } + }); + }); + }); + } + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), @@ -168,6 +228,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) @@ -181,6 +242,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) @@ -216,7 +278,8 @@ public abstract class BaseEntityImportService saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -208,6 +208,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(false) .build(), true, true); } catch (Exception e) { @@ -246,6 +247,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(config.isFindExistingEntityByName()) .build(), false, false); @@ -283,6 +285,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); VersionLoadResult result = results.get(entityType); result.setDeleted(result.getDeleted() + 1); } @@ -321,6 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(true) + .exportAttributes(true) .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java new file mode 100644 index 0000000000..dcf69c0877 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; + +@Data +public class AttributeExportData { + private String key; + private Long lastUpdateTs; + + private Boolean booleanValue; + private String strValue; + private Long longValue; + private Double doubleValue; + private String jsonValue; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 35fceafe43..542e675940 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; +import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @@ -45,5 +46,6 @@ public class EntityExportData> { private EntityType entityType; private List relations; + private Map> attributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 051745c07f..0800a1f7c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -26,4 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityExportSettings { private boolean exportRelations; + private boolean exportAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 3e18bc9f5f..564b0134e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 10971f1170..4426717fe9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -24,4 +24,5 @@ public class VersionCreateConfig implements Serializable { private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 8ed9c34694..2d68a2b7e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,5 +21,6 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; + private boolean loadAttributes; } From 09402f40acb56b7f0d14ade095c4c509cabebd36 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 16:16:24 +0300 Subject: [PATCH 087/156] UI: Refactor vc settings to repository settings. Add remove other entities confirm. --- .../server/controller/AdminController.java | 77 ++++--- .../DefaultTbRepositorySettingsService.java | 8 +- ...hMethod.java => RepositoryAuthMethod.java} | 2 +- .../data/sync/vc/RepositorySettings.java | 6 +- .../server/service/sync/vc/GitRepository.java | 14 +- ui-ngx/src/app/core/auth/auth.actions.ts | 10 +- ui-ngx/src/app/core/auth/auth.models.ts | 2 +- ui-ngx/src/app/core/auth/auth.reducer.ts | 4 +- ui-ngx/src/app/core/auth/auth.selectors.ts | 4 +- ui-ngx/src/app/core/auth/auth.service.ts | 10 +- ui-ngx/src/app/core/http/admin.service.ts | 47 ++-- .../http/entities-version-control.service.ts | 8 +- ui-ngx/src/app/core/services/menu.service.ts | 8 +- .../home/components/home-components.module.ts | 13 +- ...entity-types-version-create.component.html | 13 +- .../entity-types-version-create.component.ts | 2 + .../entity-types-version-load.component.html | 28 ++- .../vc/entity-types-version-load.component.ts | 42 +++- .../vc/entity-version-create.component.html | 3 + .../vc/entity-version-create.component.ts | 6 +- .../vc/entity-version-restore.component.html | 3 + .../vc/entity-version-restore.component.ts | 6 +- ...move-other-entities-confirm.component.html | 41 ++++ ...remove-other-entities-confirm.component.ts | 66 ++++++ ...tml => repository-settings.component.html} | 26 +-- ...css => repository-settings.component.scss} | 2 +- .../vc/repository-settings.component.ts | 216 +++++++++++++++++ .../vc/version-control-settings.component.ts | 218 ------------------ .../vc/version-control.component.html | 6 +- .../vc/version-control.component.ts | 10 +- .../home/pages/admin/admin-routing.module.ts | 10 +- .../modules/home/pages/admin/admin.module.ts | 4 +- ... repository-admin-settings.component.html} | 2 +- ...=> repository-admin-settings.component.ts} | 12 +- ui-ngx/src/app/shared/models/constants.ts | 2 +- .../src/app/shared/models/settings.models.ts | 19 +- ui-ngx/src/app/shared/models/vc.models.ts | 23 +- .../assets/locale/locale.constant-en_US.json | 19 +- 38 files changed, 592 insertions(+), 400 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/{VersionControlAuthMethod.java => RepositoryAuthMethod.java} (94%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts rename ui-ngx/src/app/modules/home/components/vc/{version-control-settings.component.html => repository-settings.component.html} (78%) rename ui-ngx/src/app/modules/home/components/vc/{version-control-settings.component.scss => repository-settings.component.scss} (96%) create mode 100644 ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts delete mode 100644 ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts rename ui-ngx/src/app/modules/home/pages/admin/{version-control-admin-settings.component.html => repository-admin-settings.component.html} (87%) rename ui-ngx/src/app/modules/home/pages/admin/{version-control-admin-settings.component.ts => repository-admin-settings.component.ts} (68%) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 67665ad8c7..ed740f73fb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -191,12 +191,12 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Get version control settings (getVersionControlSettings)", - notes = "Get the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Get repository settings (getRepositorySettings)", + notes = "Get the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vcSettings") + @GetMapping("/repositorySettings") @ResponseBody - public RepositorySettings getVersionControlSettings() throws ThingsboardException { + public RepositorySettings getRepositorySettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); @@ -209,12 +209,12 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", - notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Check repository settings exists (repositorySettingsExists)", + notes = "Check whether the repository settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vcSettings/exists") + @GetMapping("/repositorySettings/exists") @ResponseBody - public Boolean versionControlSettingsExists() throws ThingsboardException { + public Boolean repositorySettingsExists() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); return versionControlService.getVersionControlSettings(getTenantId()) != null; @@ -223,11 +223,11 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", - notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Creates or Updates the repository settings (saveRepositorySettings)", + notes = "Creates or Updates the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vcSettings") - public DeferredResult saveVersionControlSettings(@RequestBody RepositorySettings settings) throws ThingsboardException { + @PostMapping("/repositorySettings") + public DeferredResult saveRepositorySettings(@RequestBody RepositorySettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); return wrapFuture(Futures.transform(future, savedSettings -> { @@ -238,13 +238,13 @@ public class AdminController extends BaseController { }, MoreExecutors.directExecutor())); } - @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", - notes = "Deletes the version control settings." + @ApiOperation(value = "Delete repository settings (deleteRepositorySettings)", + notes = "Deletes the repository settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vcSettings", method = RequestMethod.DELETE) + @RequestMapping(value = "/repositorySettings", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) - public DeferredResult deleteVersionControlSettings() throws ThingsboardException { + public DeferredResult deleteRepositorySettings() throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.DELETE); return wrapFuture(versionControlService.deleteVersionControlSettings(getTenantId())); @@ -253,6 +253,23 @@ public class AdminController extends BaseController { } } + + @ApiOperation(value = "Check repository access (checkRepositoryAccess)", + notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings/checkAccess", method = RequestMethod.POST) + public DeferredResult checkRepositoryAccess( + @ApiParam(value = "A JSON value representing the Repository Settings.") + @RequestBody RepositorySettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + settings = checkNotNull(settings); + return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -267,8 +284,8 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control settings exists (versionControlSettingsExists)", - notes = "Check whether the version control settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", + notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @GetMapping("/vc/autoCommitSettings/exists") @ResponseBody @@ -281,8 +298,8 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Creates or Updates the version control settings (saveVersionControlSettings)", - notes = "Creates or Updates the version control settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", + notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @PostMapping("/vc/autoCommitSettings") public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { @@ -290,8 +307,8 @@ public class AdminController extends BaseController { return autoCommitSettingsService.save(getTenantId(), settings); } - @ApiOperation(value = "Delete version control settings (deleteVersionControlSettings)", - notes = "Deletes the version control settings." + @ApiOperation(value = "Delete auto commit settings (deleteAutoCommitSettings)", + notes = "Deletes the auto commit settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) @@ -305,22 +322,6 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Check version control access (checkVersionControlAccess)", - notes = "Attempts to check version control access. " + TENANT_AUTHORITY_PARAGRAPH) - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vcSettings/checkAccess", method = RequestMethod.POST) - public DeferredResult checkVersionControlAccess( - @ApiParam(value = "A JSON value representing the Entities Version Control Settings.") - @RequestBody RepositorySettings settings) throws ThingsboardException { - try { - accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - settings = checkNotNull(settings); - return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); - } catch (Exception e) { - throw handleException(e); - } - } - @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java index 608efec870..555490bd06 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -19,7 +19,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; @@ -38,10 +38,10 @@ public class DefaultTbRepositorySettingsService extends TbAbstractVersionControl public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { RepositorySettings storedSettings = get(tenantId); if (storedSettings != null) { - VersionControlAuthMethod authMethod = settings.getAuthMethod(); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + RepositoryAuthMethod authMethod = settings.getAuthMethod(); + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { settings.setPassword(storedSettings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { settings.setPrivateKey(storedSettings.getPrivateKey()); if (settings.getPrivateKeyPassword() == null) { settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java index 770a1e582c..3c0e5849a3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.common.data.sync.vc; -public enum VersionControlAuthMethod { +public enum RepositoryAuthMethod { USERNAME_PASSWORD, PRIVATE_KEY } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index c576f82298..eb1a97ff6c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -17,12 +17,8 @@ package org.thingsboard.server.common.data.sync.vc; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; @Data @JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. @@ -30,7 +26,7 @@ public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; private String repositoryUri; - private VersionControlAuthMethod authMethod; + private RepositoryAuthMethod authMethod; private String username; private String password; private String privateKeyFileName; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index dd9a67aed4..2b2802d003 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -59,7 +59,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; -import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -95,9 +95,9 @@ public class GitRepository { public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } CloneCommand cloneCommand = Git.cloneRepository() @@ -113,9 +113,9 @@ public class GitRepository { Git git = Git.open(directory); CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); @@ -124,9 +124,9 @@ public class GitRepository { public static void test(RepositorySettings settings, File directory) throws GitAPIException { CredentialsProvider credentialsProvider = null; SshdSessionFactory sshSessionFactory = null; - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); } LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 872607b822..a60720e61d 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -24,7 +24,7 @@ export enum AuthActionTypes { LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', - UPDATE_HAS_VERSION_CONTROL = '[Auth] Change Has Version Control' + UPDATE_HAS_REPOSITORY = '[Auth] Change Has Repository' } export class ActionAuthAuthenticated implements Action { @@ -55,11 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } -export class ActionAuthUpdateHasVersionControl implements Action { - readonly type = AuthActionTypes.UPDATE_HAS_VERSION_CONTROL; +export class ActionAuthUpdateHasRepository implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_REPOSITORY; - constructor(readonly payload: { hasVersionControl: boolean }) {} + constructor(readonly payload: { hasRepository: boolean }) {} } export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasVersionControl; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasRepository; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index a69b783526..33b84945aa 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,7 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; - hasVersionControl: boolean; + hasRepository: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index 9d9404524b..cace62660e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -24,7 +24,7 @@ const emptyUserAuthState: AuthPayload = { forceFullscreen: false, allowedDashboardIds: [], edgesSupportEnabled: false, - hasVersionControl: false + hasRepository: false }; export const initialState: AuthState = { @@ -55,7 +55,7 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; - case AuthActionTypes.UPDATE_HAS_VERSION_CONTROL: + case AuthActionTypes.UPDATE_HAS_REPOSITORY: return { ...state, ...action.payload}; default: diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index 7a099d5acd..4a63dbcbed 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,9 +55,9 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); -export const selectHasVersionControl = createSelector( +export const selectHasRepository = createSelector( selectAuthState, - (state: AuthState) => state.hasVersionControl + (state: AuthState) => state.hasRepository ); export function getCurrentAuthState(store: Store): AuthState { diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 6fe809955f..74f6689553 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -437,9 +437,9 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } - private loadHasVersionControl(authUser: AuthUser): Observable { + private loadHasRepository(authUser: AuthUser): Observable { if (authUser.authority === Authority.TENANT_ADMIN) { - return this.http.get('/api/admin/vcSettings/exists', defaultHttpOptions()); + return this.http.get('/api/admin/repositorySettings/exists', defaultHttpOptions()); } else { return of(false); } @@ -449,15 +449,15 @@ export class AuthService { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), - this.loadHasVersionControl(authPayload.authUser), + this.loadHasRepository(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - const hasVersionControl: boolean = data[3] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasVersionControl}; + const hasRepository: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasRepository}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 7c12bd3a5d..38f18bd598 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -15,19 +15,21 @@ /// import { Injectable } from '@angular/core'; -import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable } from 'rxjs'; +import { defaultHttpOptions, defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, - EntitiesVersionControlSettings, + RepositorySettings, MailServerSettings, SecuritySettings, TestSmsRequest, - UpdateMessage + UpdateMessage, AutoCommitSettings } from '@shared/models/settings.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { tap } from 'rxjs/operators'; +import { AuthUser } from '@shared/models/user.model'; +import { Authority } from '@shared/models/authority.enum'; @Injectable({ providedIn: 'root' @@ -68,13 +70,13 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } - public getEntitiesVersionControlSettings(config?: RequestConfig): Observable { - return this.http.get(`/api/admin/vcSettings`, defaultHttpOptionsFromConfig(config)); + public getRepositorySettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); } - public saveEntitiesVersionControlSettings(versionControlSettings: EntitiesVersionControlSettings, - config?: RequestConfig): Observable { - return this.http.post('/api/admin/vcSettings', versionControlSettings, + public saveRepositorySettings(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings', repositorySettings, defaultHttpOptionsFromConfig(config)).pipe( tap(() => { this.entitiesVersionControlService.clearBranchList(); @@ -82,17 +84,34 @@ export class AdminService { ); } - public deleteEntitiesVersionControlSettings(config?: RequestConfig) { - return this.http.delete('/api/admin/vcSettings', defaultHttpOptionsFromConfig(config)).pipe( + public deleteRepositorySettings(config?: RequestConfig) { + return this.http.delete('/api/admin/repositorySettings', defaultHttpOptionsFromConfig(config)).pipe( tap(() => { this.entitiesVersionControlService.clearBranchList(); }) ); } - public checkVersionControlAccess(versionControlSettings: EntitiesVersionControlSettings, - config?: RequestConfig): Observable { - return this.http.post('/api/admin/vcSettings/checkAccess', versionControlSettings, defaultHttpOptionsFromConfig(config)); + public checkRepositoryAccess(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings/checkAccess', repositorySettings, defaultHttpOptionsFromConfig(config)); + } + + public getAutoCommitSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); + } + + private autoCommitSettingsExists(config?: RequestConfig): Observable { + return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); + } + + public saveAutoCommitSettings(autoCommitSettings: AutoCommitSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/autoCommitSettings', autoCommitSettings, defaultHttpOptionsFromConfig(config)); + } + + public deleteAutoCommitSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/autoCommitSettings', defaultHttpOptionsFromConfig(config)); } public checkUpdates(config?: RequestConfig): Observable { diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index 7578696774..9f6f4f3403 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -17,7 +17,7 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; -import { combineLatest, Observable, of } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { BranchInfo, EntityDataDiff, EntityVersion, @@ -29,10 +29,10 @@ import { PageLink } from '@shared/models/page/page-link'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; import { EntityType } from '@shared/models/entity-type.models'; -import { createSelector, select, Store } from '@ngrx/store'; +import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { selectHasVersionControl, selectIsAuthenticated, selectIsUserLoaded } from '@core/auth/auth.selectors'; -import { catchError, combineAll, tap } from 'rxjs/operators'; +import { selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index ee81f007ee..50938e5869 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -388,9 +388,9 @@ export class MenuService { }, { id: guid(), - name: 'admin.git-settings', + name: 'admin.repository-settings', type: 'link', - path: '/settings/vc', + path: '/settings/repository', icon: 'manage_history' } ] @@ -538,9 +538,9 @@ export class MenuService { path: '/settings/resources-library' }, { - name: 'admin.git-settings', + name: 'admin.repository-settings', icon: 'manage_history', - path: '/settings/vc', + path: '/settings/repository', } ] } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index e9fc3f8c9a..613f236cf6 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -153,7 +153,7 @@ import { TenantProfileQueuesComponent } from '@home/components/profile/queue/ten import { QueueFormComponent } from '@home/components/queue/queue-form.component'; import { WidgetSettingsModule } from '@home/components/widget/lib/settings/widget-settings.module'; import { WidgetSettingsComponent } from '@home/components/widget/widget-settings.component'; -import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; import { VersionControlComponent } from '@home/components/vc/version-control.component'; import { EntityVersionsTableComponent } from '@home/components/vc/entity-versions-table.component'; import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; @@ -163,6 +163,7 @@ import { ComplexVersionCreateComponent } from '@home/components/vc/complex-versi import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-types-version-create.component'; import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; @NgModule({ declarations: @@ -287,7 +288,7 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VersionControlSettingsComponent, + RepositorySettingsComponent, VersionControlComponent, EntityVersionsTableComponent, EntityVersionCreateComponent, @@ -296,7 +297,8 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version ComplexVersionCreateComponent, EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, - ComplexVersionLoadComponent + ComplexVersionLoadComponent, + RemoveOtherEntitiesConfirmComponent ], imports: [ CommonModule, @@ -415,7 +417,7 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version DisplayWidgetTypesPanelComponent, TenantProfileQueuesComponent, QueueFormComponent, - VersionControlSettingsComponent, + RepositorySettingsComponent, VersionControlComponent, EntityVersionsTableComponent, EntityVersionCreateComponent, @@ -424,7 +426,8 @@ import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version ComplexVersionCreateComponent, EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, - ComplexVersionLoadComponent + ComplexVersionLoadComponent, + RemoveOtherEntitiesConfirmComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 9d1192c0c9..3c1dd73117 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -53,7 +53,7 @@ [allowedEntityTypes]="allowedEntityTypes(entityTypeFormGroup)">
- + version-control.sync-strategy @@ -64,9 +64,14 @@ - - {{ 'version-control.export-entity-relations' | translate }} - +
+ + {{ 'version-control.export-entity-relations' | translate }} + + + {{ 'version-control.export-entity-attributes' | translate }} + +
-
+
-
- - {{ 'version-control.remove-other-entities' | translate }} - - - {{ 'version-control.load-entities-relations' | translate }} - +
+
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.find-existing-entity-by-name' | translate }} + +
+
+ + {{ 'version-control.load-entities-relations' | translate }} + + + {{ 'version-control.load-entities-attributes' | translate }} + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 39709bebe1..40bf9ad869 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { Component, forwardRef, Input, OnInit, Renderer2, ViewContainerRef } from '@angular/core'; import { AbstractControl, ControlValueAccessor, @@ -28,11 +28,15 @@ import { Validators } from '@angular/forms'; import { PageComponent } from '@shared/components/page.component'; -import { EntityTypeVersionLoadConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes, VersionCreationResult } from '@shared/models/vc.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { TranslateService } from '@ngx-translate/core'; import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { MatCheckbox } from '@angular/material/checkbox/checkbox'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; @Component({ selector: 'tb-entity-types-version-load', @@ -64,6 +68,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On constructor(protected store: Store, private translate: TranslateService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private viewContainerRef: ViewContainerRef, private fb: FormBuilder) { super(store); } @@ -124,7 +131,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On entityType: [entityType, [Validators.required]], config: this.fb.group({ loadRelations: [config.loadRelations, []], - removeOtherEntities: [config.removeOtherEntities, []] + loadAttributes: [config.loadAttributes, []], + removeOtherEntities: [config.removeOtherEntities, []], + findExistingEntityByName: [config.findExistingEntityByName, []] }) } ); @@ -156,7 +165,9 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionLoadConfig = { loadRelations: false, - removeOtherEntities: false + loadAttributes: false, + removeOtherEntities: false, + findExistingEntityByName: false }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -194,6 +205,29 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On return res; } + onRemoveOtherEntities(removeOtherEntitiesCheckbox: MatCheckbox, entityTypeControl: AbstractControl, $event: Event) { + const removeOtherEntities: boolean = entityTypeControl.get('config.removeOtherEntities').value; + if (!removeOtherEntities) { + $event.preventDefault(); + $event.stopPropagation(); + const trigger = $('.mat-checkbox-frame', removeOtherEntitiesCheckbox._elementRef.nativeElement)[0]; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const removeOtherEntitiesConfirmPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, RemoveOtherEntitiesConfirmComponent, 'bottom', true, null, + { + onClose: (result: boolean | null) => { + removeOtherEntitiesConfirmPopover.hide(); + if (result) { + entityTypeControl.get('config').get('removeOtherEntities').patchValue(true, {emitEvent: true}); + } + } + }, {}, {}, {}, false); + } + } + } + private updateModel() { const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 5c575b93d4..b5f2fb6937 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -41,6 +41,9 @@ {{ 'version-control.export-entity-relations' | translate }} + + {{ 'version-control.export-entity-attributes' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 9f998bc2af..2ff8c03227 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -62,7 +62,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], versionName: [null, [Validators.required]], - saveRelations: [false, []] + saveRelations: [false, []], + saveAttributes: [false, []] }); } @@ -78,7 +79,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni branch: this.createVersionFormGroup.get('branch').value, versionName: this.createVersionFormGroup.get('versionName').value, config: { - saveRelations: this.createVersionFormGroup.get('saveRelations').value + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, }, type: VersionCreateRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index cb71696b4e..e84beb03dc 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -29,6 +29,9 @@ {{ 'version-control.load-entity-relations' | translate }} + + {{ 'version-control.load-entity-attributes' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 2cc615c011..ea5443e036 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -57,7 +57,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ - loadRelations: [false, []] + loadRelations: [false, []], + loadAttributes: [false, []] }); } @@ -73,7 +74,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn versionId: this.versionId, externalEntityId: this.externalEntityId, config: { - loadRelations: this.restoreFormGroup.get('loadRelations').value + loadRelations: this.restoreFormGroup.get('loadRelations').value, + loadAttributes: this.restoreFormGroup.get('loadAttributes').value }, type: VersionLoadRequestType.SINGLE_ENTITY }; diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html new file mode 100644 index 0000000000..14ae91e2f2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html @@ -0,0 +1,41 @@ + +
+
+
+
+ + + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts new file mode 100644 index 0000000000..f569d96fac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts @@ -0,0 +1,66 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-remove-other-entities-confirm', + templateUrl: './remove-other-entities-confirm.component.html', + styleUrls: [] +}) +export class RemoveOtherEntitiesConfirmComponent extends PageComponent implements OnInit { + + @Input() + onClose: (result: boolean | null) => void; + + confirmFormGroup: FormGroup; + + removeOtherEntitiesConfirmText: SafeHtml; + + removeOtherEntitiesVerificationText = 'remove other entities'; + + constructor(protected store: Store, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + this.removeOtherEntitiesConfirmText = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.remove-other-entities-confirm-text')); + } + + ngOnInit(): void { + this.confirmFormGroup = this.fb.group({ + verification: [null, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + confirm(): void { + if (this.onClose) { + this.onClose(true); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html similarity index 78% rename from ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html rename to ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html index b48162d06f..bdab6c061b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -16,24 +16,24 @@ -->
- +
- admin.git-repository-settings + admin.repository-settings -
+
-
+
admin.repository-url - + admin.repository-url-required @@ -46,12 +46,12 @@ admin.auth-method - - {{versionControlAuthMethodTranslations.get(method) | translate}} + + {{repositoryAuthMethodTranslations.get(method) | translate}} -
+
common.username
-
+
+ (fileNameChanged)="repositorySettingsForm.get('privateKeyFileName').patchValue($event)"> @@ -96,10 +96,10 @@ -
diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss similarity index 96% rename from ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss rename to ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss index d1d55faf19..77ae0ffd8f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss @@ -14,7 +14,7 @@ * limitations under the License. */ :host { - mat-card.vc-settings { + mat-card.repository-settings { margin: 8px; } .fields-group { diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts new file mode 100644 index 0000000000..c73cf0e632 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -0,0 +1,216 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + RepositorySettings, + RepositoryAuthMethod, + repositoryAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; +import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { catchError, mergeMap, take } from 'rxjs/operators'; +import { of } from 'rxjs'; + +@Component({ + selector: 'tb-repository-settings', + templateUrl: './repository-settings.component.html', + styleUrls: ['./repository-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class RepositorySettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; + + repositorySettingsForm: FormGroup; + settings: RepositorySettings = null; + + repositoryAuthMethod = RepositoryAuthMethod; + repositoryAuthMethods = Object.values(RepositoryAuthMethod); + repositoryAuthMethodTranslations = repositoryAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.repositorySettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: ['main', []], + authMethod: [RepositoryAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.repositorySettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.repositorySettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.store.pipe( + select(selectHasRepository), + take(1), + mergeMap((hasRepository) => { + if (hasRepository) { + return this.adminService.getRepositorySettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + if (this.settings != null) { + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + } + }); + } + + checkAccess(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.checkRepositoryAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-repository-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.saveRepositorySettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: true })); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-repository-settings-title', ), + this.translate.instant('admin.delete-repository-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteRepositorySettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.repositorySettingsForm.reset({ defaultBranch: 'main', authMethod: RepositoryAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: false })); + } + ); + } + }); + } + + changePasswordChanged() { + if (this.changePassword) { + this.repositorySettingsForm.get('password').patchValue(''); + this.repositorySettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').patchValue(''); + this.repositorySettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: RepositoryAuthMethod = this.repositorySettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.repositorySettingsForm.get('privateKeyFileName').value; + if (authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.repositorySettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.repositorySettingsForm.get('password').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('password').disable({emitEvent}); + } + this.repositorySettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.repositorySettingsForm.get('privateKey').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.repositorySettingsForm.get('username').disable({emitEvent}); + this.repositorySettingsForm.get('password').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.repositorySettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.repositorySettingsForm.get('privateKey').clearValidators(); + } else { + this.repositorySettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.repositorySettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts deleted file mode 100644 index 3462fbb728..0000000000 --- a/ui-ngx/src/app/modules/home/components/vc/version-control-settings.component.ts +++ /dev/null @@ -1,218 +0,0 @@ -/// -/// Copyright © 2016-2022 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, Input, OnInit } from '@angular/core'; -import { PageComponent } from '@shared/components/page.component'; -import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; -import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; -import { select, Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { AdminService } from '@core/http/admin.service'; -import { - EntitiesVersionControlSettings, - VersionControlAuthMethod, - versionControlAuthMethodTranslationMap -} from '@shared/models/settings.models'; -import { ActionNotificationShow } from '@core/notification/notification.actions'; -import { TranslateService } from '@ngx-translate/core'; -import { isNotEmptyStr } from '@core/utils'; -import { DialogService } from '@core/services/dialog.service'; -import { ActionSettingsChangeLanguage } from '@core/settings/settings.actions'; -import { ActionAuthUpdateHasVersionControl } from '@core/auth/auth.actions'; -import { selectHasVersionControl } from '@core/auth/auth.selectors'; -import { catchError, mergeMap, take } from 'rxjs/operators'; -import { of } from 'rxjs'; - -@Component({ - selector: 'tb-version-control-settings', - templateUrl: './version-control-settings.component.html', - styleUrls: ['./version-control-settings.component.scss', './../../pages/admin/settings-card.scss'] -}) -export class VersionControlSettingsComponent extends PageComponent implements OnInit { - - @Input() - detailsMode = false; - - versionControlSettingsForm: FormGroup; - settings: EntitiesVersionControlSettings = null; - - versionControlAuthMethod = VersionControlAuthMethod; - versionControlAuthMethods = Object.values(VersionControlAuthMethod); - versionControlAuthMethodTranslations = versionControlAuthMethodTranslationMap; - - showChangePassword = false; - changePassword = false; - - showChangePrivateKeyPassword = false; - changePrivateKeyPassword = false; - - constructor(protected store: Store, - private adminService: AdminService, - private dialogService: DialogService, - private translate: TranslateService, - public fb: FormBuilder) { - super(store); - } - - ngOnInit() { - this.versionControlSettingsForm = this.fb.group({ - repositoryUri: [null, [Validators.required]], - defaultBranch: ['main', []], - authMethod: [VersionControlAuthMethod.USERNAME_PASSWORD, [Validators.required]], - username: [null, []], - password: [null, []], - privateKeyFileName: [null, [Validators.required]], - privateKey: [null, []], - privateKeyPassword: [null, []] - }); - this.updateValidators(false); - this.versionControlSettingsForm.get('authMethod').valueChanges.subscribe(() => { - this.updateValidators(true); - }); - this.versionControlSettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { - this.updateValidators(false); - }); - this.store.pipe( - select(selectHasVersionControl), - take(1), - mergeMap((hasVersionControl) => { - if (hasVersionControl) { - return this.adminService.getEntitiesVersionControlSettings({ignoreErrors: true}).pipe( - catchError(() => of(null)) - ); - } else { - return of(null); - } - }) - ).subscribe( - (settings) => { - this.settings = settings; - if (this.settings != null) { - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - } else { - this.showChangePrivateKeyPassword = true; - } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); - } - }); - } - - checkAccess(): void { - const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; - this.adminService.checkVersionControlAccess(settings).subscribe(() => { - this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-vc-access-success'), - type: 'success' })); - }); - } - - save(): void { - const settings: EntitiesVersionControlSettings = this.versionControlSettingsForm.value; - this.adminService.saveEntitiesVersionControlSettings(settings).subscribe( - (savedSettings) => { - this.settings = savedSettings; - if (this.settings.authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.showChangePassword = true; - this.changePassword = false; - } else { - this.showChangePrivateKeyPassword = true; - this.changePrivateKeyPassword = false; - } - this.versionControlSettingsForm.reset(this.settings); - this.updateValidators(false); - this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: true })); - } - ); - } - - delete(formDirective: FormGroupDirective): void { - this.dialogService.confirm( - this.translate.instant('admin.delete-git-settings-title', ), - this.translate.instant('admin.delete-git-settings-text'), null, - this.translate.instant('action.delete') - ).subscribe((data) => { - if (data) { - this.adminService.deleteEntitiesVersionControlSettings().subscribe( - () => { - this.settings = null; - this.showChangePassword = false; - this.changePassword = false; - this.showChangePrivateKeyPassword = false; - this.changePrivateKeyPassword = false; - formDirective.resetForm(); - this.versionControlSettingsForm.reset({ defaultBranch: 'main', authMethod: VersionControlAuthMethod.USERNAME_PASSWORD }); - this.updateValidators(false); - this.store.dispatch(new ActionAuthUpdateHasVersionControl({ hasVersionControl: false })); - } - ); - } - }); - } - - changePasswordChanged() { - if (this.changePassword) { - this.versionControlSettingsForm.get('password').patchValue(''); - this.versionControlSettingsForm.get('password').markAsDirty(); - } - this.updateValidators(false); - } - - changePrivateKeyPasswordChanged() { - if (this.changePrivateKeyPassword) { - this.versionControlSettingsForm.get('privateKeyPassword').patchValue(''); - this.versionControlSettingsForm.get('privateKeyPassword').markAsDirty(); - } - this.updateValidators(false); - } - - updateValidators(emitEvent?: boolean): void { - const authMethod: VersionControlAuthMethod = this.versionControlSettingsForm.get('authMethod').value; - const privateKeyFileName: string = this.versionControlSettingsForm.get('privateKeyFileName').value; - if (authMethod === VersionControlAuthMethod.USERNAME_PASSWORD) { - this.versionControlSettingsForm.get('username').enable({emitEvent}); - if (this.changePassword || !this.showChangePassword) { - this.versionControlSettingsForm.get('password').enable({emitEvent}); - } else { - this.versionControlSettingsForm.get('password').disable({emitEvent}); - } - this.versionControlSettingsForm.get('privateKeyFileName').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKey').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); - } else { - this.versionControlSettingsForm.get('username').disable({emitEvent}); - this.versionControlSettingsForm.get('password').disable({emitEvent}); - this.versionControlSettingsForm.get('privateKeyFileName').enable({emitEvent}); - this.versionControlSettingsForm.get('privateKey').enable({emitEvent}); - if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { - this.versionControlSettingsForm.get('privateKeyPassword').enable({emitEvent}); - } else { - this.versionControlSettingsForm.get('privateKeyPassword').disable({emitEvent}); - } - if (isNotEmptyStr(privateKeyFileName)) { - this.versionControlSettingsForm.get('privateKey').clearValidators(); - } else { - this.versionControlSettingsForm.get('privateKey').setValidators([Validators.required]); - } - } - this.versionControlSettingsForm.get('username').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('password').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); - this.versionControlSettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); - } - -} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index e5e650ac85..3cf10ab168 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -15,9 +15,9 @@ limitations under the License. --> - - + + (); - hasVersionControl$ = this.store.pipe(select(selectHasVersionControl)); + hasRepository$ = this.store.pipe(select(selectHasRepository)); constructor(private store: Store) { @@ -61,7 +61,7 @@ export class VersionControlComponent implements OnInit, HasConfirmForm { } confirmForm(): FormGroup { - return this.versionControlSettingsComponent?.versionControlSettingsForm; + return this.repositorySettingsComponent?.repositorySettingsForm; } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index 958e7b1c9b..16baa3aa89 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,7 +33,7 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; -import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -225,14 +225,14 @@ const routes: Routes = [ ] }, { - path: 'vc', - component: VersionControlAdminSettingsComponent, + path: 'repository', + component: RepositoryAdminSettingsComponent, canDeactivate: [ConfirmOnExitGuard], data: { auth: [Authority.TENANT_ADMIN], - title: 'admin.git-settings', + title: 'admin.repository-settings', breadcrumb: { - label: 'admin.git-settings', + label: 'admin.repository-settings', icon: 'manage_history' } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 51612270df..85e3d04767 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,7 +29,7 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; -import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version-control-admin-settings.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; @NgModule({ declarations: @@ -43,7 +43,7 @@ import { VersionControlAdminSettingsComponent } from '@home/pages/admin/version- HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, - VersionControlAdminSettingsComponent + RepositoryAdminSettingsComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html similarity index 87% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html rename to ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html index 869067e1fb..7b408fc93a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html @@ -15,4 +15,4 @@ limitations under the License. --> - + diff --git a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts similarity index 68% rename from ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts rename to ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts index a203f64426..1ee0a7288a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/version-control-admin-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts @@ -20,16 +20,16 @@ import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { FormGroup } from '@angular/forms'; -import { VersionControlSettingsComponent } from '@home/components/vc/version-control-settings.component'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; @Component({ - selector: 'tb-version-control-admin-settings', - templateUrl: './version-control-admin-settings.component.html', + selector: 'tb-repository-admin-settings', + templateUrl: './repository-admin-settings.component.html', styleUrls: [] }) -export class VersionControlAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class RepositoryAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { - @ViewChild('versionControlSettingsComponent') versionControlSettingsComponent: VersionControlSettingsComponent; + @ViewChild('repositorySettingsComponent') repositorySettingsComponent: RepositorySettingsComponent; constructor(protected store: Store) { super(store); @@ -39,6 +39,6 @@ export class VersionControlAdminSettingsComponent extends PageComponent implemen } confirmForm(): FormGroup { - return this.versionControlSettingsComponent?.versionControlSettingsForm; + return this.repositorySettingsComponent?.repositorySettingsForm; } } diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 6297809522..ff16183509 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -135,7 +135,7 @@ export const HelpLinks = { ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', queue: helpBaseUrl + '/docs/user-guide/queue', - versionControlSettings: helpBaseUrl + '/docs/user-guide/ui/version-control-settings' + repositorySettings: helpBaseUrl + '/docs/user-guide/ui/repository-settings' } }; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 8a1b316001..298281da11 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -16,6 +16,7 @@ import { ValidatorFn } from '@angular/forms'; import { isNotEmptyStr, isNumber } from '@core/utils'; +import { VersionCreateConfig } from '@shared/models/vc.models'; export const smtpPortPattern: RegExp = /^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/; @@ -397,23 +398,29 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid return smsProviderConfiguration; } -export enum VersionControlAuthMethod { +export enum RepositoryAuthMethod { USERNAME_PASSWORD = 'USERNAME_PASSWORD', PRIVATE_KEY = 'PRIVATE_KEY' } -export const versionControlAuthMethodTranslationMap = new Map([ - [VersionControlAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], - [VersionControlAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +export const repositoryAuthMethodTranslationMap = new Map([ + [RepositoryAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [RepositoryAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] ]); -export interface EntitiesVersionControlSettings { +export interface RepositorySettings { repositoryUri: string; defaultBranch: string; - authMethod: VersionControlAuthMethod; + authMethod: RepositoryAuthMethod; username: string; password: string; privateKeyFileName: string; privateKey: string; privateKeyPassword: string; } + +export interface AutoVersionCreateConfig extends VersionCreateConfig { + branch: string; +} + +export type AutoCommitSettings = {[entityType: string]: AutoVersionCreateConfig}; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index 0ad52d2e0b..cd1cba93ed 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -32,6 +32,7 @@ export const exportableEntityTypes: Array = [ export interface VersionCreateConfig { saveRelations: boolean; + saveAttributes: boolean; } export enum VersionCreateRequestType { @@ -81,6 +82,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: res[entityType] = { syncStrategy: null, saveRelations: false, + saveAttributes: false, allEntities: true, entityIds: [] }; @@ -90,6 +92,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: export interface VersionLoadConfig { loadRelations: boolean; + loadAttributes: boolean; } export enum VersionLoadRequestType { @@ -111,6 +114,7 @@ export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { removeOtherEntities: boolean; + findExistingEntityByName: boolean; } export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { @@ -123,7 +127,9 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En for (const entityType of exportableEntityTypes) { res[entityType] = { loadRelations: false, - removeOtherEntities: false + loadAttributes: false, + removeOtherEntities: false, + findExistingEntityByName: false }; } return res; @@ -176,18 +182,5 @@ export interface EntityDataDiff { } export function entityExportDataToJsonString(data: EntityExportData): string { - if (!data.relations) { - data.relations = []; - } - const allKeys = new Set(); - JSON.stringify(data, (key, value) => (allKeys.add(key), value)); - return JSON.stringify(data, Array.from(allKeys).sort((key1, key2) => { - if (key1 === 'relations') { - return 1; - } else if (key2 === 'relations') { - return -1; - } else { - return 0; - } - }), 4); + return JSON.stringify(data, null, 4); } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 4a4e228cc2..8a2f63cb8f 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -58,7 +58,8 @@ "next-with-label": "Next: {{label}}", "read-more": "Read more", "hide": "Hide", - "restore": "Restore" + "restore": "Restore", + "confirm": "Confirm" }, "aggregation": { "aggregation": "Aggregation", @@ -321,8 +322,7 @@ "queue-submit-strategy": "Submit strategy", "queue-processing-strategy": "Processing strategy", "queue-configuration": "Queue configuration", - "git-settings": "Git settings", - "git-repository-settings": "Git repository settings", + "repository-settings": "Repository settings", "repository-url": "Repository URL", "repository-url-required": "Repository URL is required.", "default-branch": "Default branch name", @@ -338,9 +338,9 @@ "enter-passphrase": "Enter passphrase", "change-passphrase": "Change passphrase", "check-access": "Check access", - "check-vc-access-success": "Git repository access successfully verified!", - "delete-git-settings-title": "Are you sure you want to delete git settings?", - "delete-git-settings-text": "Be careful, after the confirmation the git settings will be removed and git synchronization feature will be unavailable." + "check-repository-access-success": "Repository access successfully verified!", + "delete-repository-settings-title": "Are you sure you want to delete repository settings?", + "delete-repository-settings-text": "Be careful, after the confirmation the repository settings will be removed and version control feature will be unavailable." }, "alarm": { "alarm": "Alarm", @@ -3121,6 +3121,7 @@ "version-name-required": "Version name is required", "author": "Author", "export-entity-relations": "Export entity relations", + "export-entity-attributes": "Export entity attributes", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3133,6 +3134,7 @@ "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-entity-relations": "Load entity relations", + "load-entity-attributes": "Load entity attributes", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3151,12 +3153,15 @@ "remove-all": "Remove all", "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", "load-entities-relations": "Load entities relations", + "load-entities-attributes": "Load entities attributes", "remove-other-entities": "Remove other entities", + "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", "no-entities-restored": "No entities restored", "created": "{{created}} created", "updated": "{{updated}} updated", - "deleted": "{{deleted}} deleted" + "deleted": "{{deleted}} deleted", + "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm." }, "widget": { "widget-library": "Widgets Library", From 59325226ac5020227644afee6a2641bec462e8b2 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 16:38:10 +0300 Subject: [PATCH 088/156] Sort order and diff fixes --- .../DefaultEntitiesVersionControlService.java | 19 +++++++++-------- .../DefaultGitVersionControlQueueService.java | 4 ++-- .../common/data/sync/ie/EntityExportData.java | 21 +++++++++++++++++++ .../thingsboard/common/util/JacksonUtil.java | 10 ++++++--- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index fb68967661..05fbef899b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -328,16 +328,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityId externalId = ((ExportableEntity) entity).getExternalId(); if (externalId == null) externalId = entityId; - EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() - .exportRelations(true) - .exportAttributes(true) - .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), - otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion), - JacksonUtil.toPrettyString(otherVersion)), rawDiff -> { - return new EntityDataDiff(currentVersion, otherVersion, rawDiff); - }, MoreExecutors.directExecutor()), MoreExecutors.directExecutor()); + otherVersion -> { + EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + .exportRelations(otherVersion.getRelations() != null) + .exportAttributes(otherVersion.getAttributes() != null) + .build()); + return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), + rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 7019d6adb5..095c56f143 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -109,7 +109,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu SettableFuture future = SettableFuture.create(); String path = getRelativePath(entityData.getEntityType(), entityData.getEntity().getId()); - String entityDataJson = JacksonUtil.toPrettyString(entityData); + String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); registerAndSend(commit, builder -> builder.setCommitRequest( buildCommitRequest(commit).setAddMsg( @@ -130,7 +130,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu buildCommitRequest(commit).setDeleteMsg( TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() ).build() - ).build(), wrap(commit.getFuture(), null)); + ).build(), wrap(future, null)); return future; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 542e675940..0fbdae756e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -28,6 +28,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; +import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; @@ -41,6 +43,15 @@ import java.util.Map; @Data public class EntityExportData> { + public static final Comparator relationsComparator = Comparator + .comparing(EntityRelation::getFrom, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTo, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTypeGroup) + .thenComparing(EntityRelation::getType); + + public static final Comparator attrComparator = Comparator + .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + @JsonTbEntity private E entity; private EntityType entityType; @@ -48,4 +59,14 @@ public class EntityExportData> { private List relations; private Map> attributes; + public EntityExportData sort() { + if (relations != null && !relations.isEmpty()) { + relations.sort(relationsComparator); + } + if (attributes != null && !attributes.isEmpty()) { + attributes.values().forEach(list -> list.sort(attrComparator)); + } + return this; + } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index b6e2b54479..d622801814 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -18,8 +18,10 @@ package org.thingsboard.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; @@ -31,8 +33,10 @@ import java.util.Arrays; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public static final ObjectMapper PRETTY_JSON_MAPPER = new ObjectMapper() - .enable(SerializationFeature.INDENT_OUTPUT); + public static final ObjectMapper PRETTY_SORTED_JSON_MAPPER = JsonMapper.builder().enable(SerializationFeature.INDENT_OUTPUT) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) + .build(); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -99,7 +103,7 @@ public class JacksonUtil { public static String toPrettyString(Object o) { try { - return PRETTY_JSON_MAPPER.writeValueAsString(o); + return PRETTY_SORTED_JSON_MAPPER.writeValueAsString(o); } catch (JsonProcessingException e) { throw new RuntimeException(e); } From 7bd51cb4cec451b38e0d348f61ed046a4b781e14 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 17:13:50 +0300 Subject: [PATCH 089/156] Public Customer import fix. No commit when no changes --- .../importing/impl/CustomerImportService.java | 6 +++++- .../DefaultGitVersionControlQueueService.java | 18 ++++++++++-------- .../data/sync/vc/RepositorySettings.java | 1 - .../DefaultClusterVersionControlService.java | 18 +++++++++++------- .../sync/vc/DefaultGitRepositoryService.java | 9 +++++---- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index 2a7570e644..aec96b2e47 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -42,7 +42,11 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { - return customerService.saveCustomer(customer); + if (customer.isPublic()) { + return customerService.findOrCreatePublicCustomer(tenantId); + } else { + return customerService.saveCustomer(customer); + } } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 095c56f143..c87789a096 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -152,7 +152,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch), + .setBranchName(branch), pageLink ).build()); } @@ -162,8 +162,8 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch) - .setEntityType(entityType.name()), + .setBranchName(branch) + .setEntityType(entityType.name()), pageLink ).build()); } @@ -173,10 +173,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return listVersions(tenantId, applyPageLinkParameters( ListVersionsRequestMsg.newBuilder() - .setBranchName(branch) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), pageLink ).build()); } @@ -354,7 +354,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu } else if (vcResponseMsg.hasCommitResponse()) { var commitResponse = vcResponseMsg.getCommitResponse(); var commitResult = new VersionCreationResult(); - commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + if (commitResponse.getTs() > 0) { + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + } commitResult.setAdded(commitResponse.getAdded()); commitResult.setRemoved(commitResponse.getRemoved()); commitResult.setModified(commitResponse.getModified()); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java index eb1a97ff6c..50e23a1dba 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -21,7 +21,6 @@ import lombok.Data; import java.io.Serializable; @Data -@JsonIgnoreProperties(ignoreUnknown = true) // temporary to make sure no need to wipe db during development. public class RepositorySettings implements Serializable { private static final long serialVersionUID = -3211552851889198721L; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index 6ff7347be9..b28618372e 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -457,14 +457,18 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { - reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(CommitResponseMsg.newBuilder() - .setTs(result.getVersion().getTimestamp()) - .setCommitId(result.getVersion().getId()) - .setName(result.getVersion().getName()) - .setAuthor(result.getVersion().getAuthor()) - .setAdded(result.getAdded()) + var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded()) .setModified(result.getModified()) - .setRemoved(result.getRemoved()))); + .setRemoved(result.getRemoved()); + + if (result.getVersion() != null) { + responseBuilder.setTs(result.getVersion().getTimestamp()) + .setCommitId(result.getVersion().getId()) + .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()); + } + + reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(responseBuilder)); } private void reply(VersionControlRequestCtx ctx, Optional e) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java index 6db0a49d4f..99dbebc3c9 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -117,10 +117,11 @@ public class DefaultGitRepositoryService implements GitRepositoryService { result.setModified(status.getModified().size()); result.setRemoved(status.getRemoved().size()); - GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); - repository.push(commit.getWorkingBranch(), commit.getBranch()); - - result.setVersion(toVersion(gitCommit)); + if (result.getAdded() > 0 || result.getModified() > 0 || result.getRemoved() > 0) { + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); + repository.push(commit.getWorkingBranch(), commit.getBranch()); + result.setVersion(toVersion(gitCommit)); + } return result; } catch (GitAPIException gitAPIException) { //TODO: analyze and return meaningful exceptions that we can show to the client; From 1610222b72528ae7399bfc82167210f9171d5764 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 17:27:53 +0300 Subject: [PATCH 090/156] Export/import api rate limiting --- .../apiusage/DefaultRateLimitService.java | 76 +++++++++++++++++++ .../service/apiusage/RateLimitService.java | 26 +++++++ .../DefaultEntitiesExportImportService.java | 11 +++ .../DefaultTenantProfileConfiguration.java | 5 +- .../server/common/msg/tools/TbRateLimits.java | 8 +- 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java new file mode 100644 index 0000000000..55b23a99cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +public class DefaultRateLimitService implements RateLimitService { + + private final TbTenantProfileCache tenantProfileCache; + + private final Map> rateLimits = new ConcurrentHashMap<>(); + + @Override + public boolean checkEntityExportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit); + } + + @Override + public boolean checkEntityImportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit); + } + + private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function rateLimitConfigExtractor) { + String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration() + .map(rateLimitConfigExtractor).orElse(null); + + Map rateLimits = this.rateLimits.get(rateLimitsKey); + if (StringUtils.isEmpty(rateLimitConfig)) { + if (rateLimits != null) { + rateLimits.remove(tenantId); + if (rateLimits.isEmpty()) { + this.rateLimits.remove(rateLimitsKey); + } + } + return true; + } + + if (rateLimits == null) { + rateLimits = new ConcurrentHashMap<>(); + this.rateLimits.put(rateLimitsKey, rateLimits); + } + TbRateLimits rateLimit = rateLimits.get(tenantId); + if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) { + rateLimit = new TbRateLimits(rateLimitConfig); + rateLimits.put(tenantId, rateLimit); + } + + return rateLimit.tryConsume(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java new file mode 100644 index 0000000000..d3d4244ca1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface RateLimitService { + + boolean checkEntityExportLimit(TenantId tenantId); + + boolean checkEntityImportLimit(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 777e84f348..859f3b9da2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.sync.ThrowingRunnable; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.apiusage.RateLimitService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; @@ -55,6 +57,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); + private final RateLimitService rateLimitService; + protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE @@ -63,6 +67,10 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); @@ -73,6 +81,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override public , I extends EntityId> EntityImportResult importEntity(SecurityUser user, EntityExportData exportData, EntityImportSettings importSettings, boolean saveReferences, boolean sendEvents) throws ThingsboardException { + if (!rateLimitService.checkEntityImportLimit(user.getTenantId())) { + throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { throw new DataValidationException("Invalid entity data"); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 15eb429805..9953f7a3aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,6 +44,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String tenantEntityExportRateLimit; + private String tenantEntityImportRateLimit; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java index afcbe1b3b9..9ca13b295d 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/tools/TbRateLimits.java @@ -27,6 +27,7 @@ import java.time.Duration; */ public class TbRateLimits { private final LocalBucket bucket; + private final String configuration; public TbRateLimits(String limitsConfiguration) { LocalBucketBuilder builder = Bucket4j.builder(); @@ -42,8 +43,7 @@ public class TbRateLimits { } else { throw new IllegalArgumentException("Failed to parse rate limits configuration: " + limitsConfiguration); } - - + this.configuration = limitsConfiguration; } public boolean tryConsume() { @@ -54,4 +54,8 @@ public class TbRateLimits { return bucket.tryConsume(number); } + public String getConfiguration() { + return configuration; + } + } From ac0b1a98794b05ff713ed45568d5eca94a1cdfa9 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Tue, 31 May 2022 17:46:23 +0300 Subject: [PATCH 091/156] getEntityDataInfo API call --- .../EntitiesVersionControlController.java | 13 +++++++++ .../DefaultEntitiesVersionControlService.java | 8 ++++++ .../vc/EntitiesVersionControlService.java | 3 ++ .../common/data/sync/vc/EntityDataInfo.java | 28 +++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index bf7011867d..36aa6fd961 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -40,6 +40,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; @@ -237,6 +238,18 @@ public class EntitiesVersionControlController extends BaseController { } } + @GetMapping("/info/{versionId}/{entityType}/{internalEntityUuid}") + public DeferredResult getEntityDataInfo(@PathVariable String versionId, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid) throws ThingsboardException { + try { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") public DeferredResult compareEntityDataToVersion(@PathVariable String branch, @PathVariable EntityType entityType, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 05fbef899b..cca21929c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -341,6 +342,13 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, MoreExecutors.directExecutor()); } + @Override + public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { + return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), + entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor()); + } + + @Override public ListenableFuture> listBranches(TenantId tenantId) throws Exception { return gitServiceQueue.listBranches(tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 447ae1d021..88c23616b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; @@ -62,4 +63,6 @@ public interface EntitiesVersionControlService { ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; + + ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java new file mode 100644 index 0000000000..ab548b1cfc --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityDataInfo { + boolean hasRelations; + boolean hasAttributes; +} From 9eb32add2bf5ebb330d14bfd23ce74d9f30df6cd Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 18:11:24 +0300 Subject: [PATCH 092/156] UI for export/import api rate limiting --- .../default-tenant-profile-configuration.component.html | 8 ++++++++ .../default-tenant-profile-configuration.component.ts | 2 ++ ui-ngx/src/app/shared/models/tenant.model.ts | 3 +++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 ++ 4 files changed, 15 insertions(+) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html index fef4d4cef5..08c8df06a0 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.html @@ -280,4 +280,12 @@ tenant-profile.transport-device-telemetry-data-points-rate-limit + + tenant-profile.tenant-entity-export-rate-limit + + + + tenant-profile.tenant-entity-import-rate-limit + + diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts index ddab7f3aae..6f79c34279 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant/default-tenant-profile-configuration.component.ts @@ -67,6 +67,8 @@ export class DefaultTenantProfileConfigurationComponent implements ControlValueA transportDeviceMsgRateLimit: [null, []], transportDeviceTelemetryMsgRateLimit: [null, []], transportDeviceTelemetryDataPointsRateLimit: [null, []], + tenantEntityExportRateLimit: [null, []], + tenantEntityImportRateLimit: [null, []], maxTransportMessages: [null, [Validators.required, Validators.min(0)]], maxTransportDataPoints: [null, [Validators.required, Validators.min(0)]], maxREExecutions: [null, [Validators.required, Validators.min(0)]], diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 33d1367c01..ea1ff3fdc5 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -41,6 +41,9 @@ export interface DefaultTenantProfileConfiguration { transportDeviceTelemetryMsgRateLimit?: string; transportDeviceTelemetryDataPointsRateLimit?: string; + tenantEntityExportRateLimit?: string; + tenantEntityImportRateLimit?: string; + maxTransportMessages: number; maxTransportDataPoints: number; maxREExecutions: number; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index db52c04ac1..90193b1499 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2944,6 +2944,8 @@ "transport-device-msg-rate-limit": "Transport device messages rate limit.", "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", + "tenant-entity-export-rate-limit": "Entity version creation rate limit", + "tenant-entity-import-rate-limit": "Entity version load rate limit", "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", "max-transport-messages-required": "Maximum number of transport messages is required.", "max-transport-messages-range": "Maximum number of transport messages can't be negative", From 7841c5ff6fa27ccbc80544d2ef222a63a0b3d19e Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 31 May 2022 18:42:33 +0300 Subject: [PATCH 093/156] UI: Implement auto-commit settings --- .../server/controller/AdminController.java | 8 +- ui-ngx/src/app/core/http/admin.service.ts | 2 +- ui-ngx/src/app/core/services/menu.service.ts | 14 +- .../home/components/home-components.module.ts | 7 +- .../vc/auto-commit-settings.component.html | 125 +++++++++++ .../vc/auto-commit-settings.component.scss | 74 ++++++ .../vc/auto-commit-settings.component.ts | 210 ++++++++++++++++++ ...entity-types-version-create.component.html | 4 +- .../entity-types-version-create.component.ts | 2 +- .../entity-types-version-load.component.html | 4 +- .../vc/entity-types-version-load.component.ts | 2 +- .../home/pages/admin/admin-routing.module.ts | 14 ++ .../modules/home/pages/admin/admin.module.ts | 4 +- .../auto-commit-admin-settings.component.html | 23 ++ .../auto-commit-admin-settings.component.ts | 51 +++++ .../vc/branch-autocomplete.component.html | 4 +- .../vc/branch-autocomplete.component.ts | 3 + ui-ngx/src/app/shared/models/constants.ts | 3 +- .../assets/locale/locale.constant-en_US.json | 16 +- 19 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index ed740f73fb..470ec738af 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -273,7 +273,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vc/autoCommitSettings") + @GetMapping("/autoCommitSettings") @ResponseBody public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { try { @@ -287,7 +287,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @GetMapping("/vc/autoCommitSettings/exists") + @GetMapping("/autoCommitSettings/exists") @ResponseBody public Boolean autoCommitSettingsExists() throws ThingsboardException { try { @@ -301,7 +301,7 @@ public class AdminController extends BaseController { @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @PostMapping("/vc/autoCommitSettings") + @PostMapping("/autoCommitSettings") public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); return autoCommitSettingsService.save(getTenantId(), settings); @@ -311,7 +311,7 @@ public class AdminController extends BaseController { notes = "Deletes the auto commit settings." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAuthority('TENANT_ADMIN')") - @RequestMapping(value = "/vc/autoCommitSettings", method = RequestMethod.DELETE) + @RequestMapping(value = "/autoCommitSettings", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) public void deleteAutoCommitSettings() throws ThingsboardException { try { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 38f18bd598..9b8c3b76b9 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -101,7 +101,7 @@ export class AdminService { return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); } - private autoCommitSettingsExists(config?: RequestConfig): Observable { + public autoCommitSettingsExists(config?: RequestConfig): Observable { return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 50938e5869..af62bf7918 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -369,7 +369,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '120px', + height: '160px', icon: 'settings', pages: [ { @@ -392,6 +392,13 @@ export class MenuService { type: 'link', path: '/settings/repository', icon: 'manage_history' + }, + { + id: guid(), + name: 'admin.auto-commit-settings', + type: 'link', + path: '/settings/auto-commit', + icon: 'settings_backup_restore' } ] } @@ -541,6 +548,11 @@ export class MenuService { name: 'admin.repository-settings', icon: 'manage_history', path: '/settings/repository', + }, + { + name: 'admin.auto-commit-settings', + icon: 'settings_backup_restore', + path: '/settings/auto-commit' } ] } diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 613f236cf6..153a8d9759 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -164,6 +164,7 @@ import { EntityTypesVersionCreateComponent } from '@home/components/vc/entity-ty import { EntityTypesVersionLoadComponent } from '@home/components/vc/entity-types-version-load.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; +import { AutoCommitSettingsComponent } from '@home/components/vc/auto-commit-settings.component'; @NgModule({ declarations: @@ -298,7 +299,8 @@ import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove- EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, ComplexVersionLoadComponent, - RemoveOtherEntitiesConfirmComponent + RemoveOtherEntitiesConfirmComponent, + AutoCommitSettingsComponent ], imports: [ CommonModule, @@ -427,7 +429,8 @@ import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove- EntityTypesVersionCreateComponent, EntityTypesVersionLoadComponent, ComplexVersionLoadComponent, - RemoveOtherEntitiesConfirmComponent + RemoveOtherEntitiesConfirmComponent, + AutoCommitSettingsComponent ], providers: [ WidgetComponentService, diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html new file mode 100644 index 0000000000..3e668b54e6 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -0,0 +1,125 @@ + +
+ + +
+ admin.auto-commit-settings + +
+
+ + + +
+ + +
+ admin.auto-commit-entities +
+
+
+ + +
+ +
+
+
+ + + +
+ + +
+ +
+
+ + +
+ + +
+
+
+ + {{ 'version-control.export-entity-relations' | translate }} + + + {{ 'version-control.export-entity-attributes' | translate }} + +
+
+
+ + +
+
+
+ admin.no-auto-commit-entities-prompt +
+
+ + + +
+
+
+
+ + + +
+ + + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss new file mode 100644 index 0000000000..ea8017b9f1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.auto-commit-settings { + margin: 8px; + .mat-divider { + position: relative; + } + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + tb-branch-autocomplete { + min-width: 200px; + max-width: 200px; + display: block; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts new file mode 100644 index 0000000000..8ac301338e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -0,0 +1,210 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { AbstractControl, FormArray, FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { AutoCommitSettings, AutoVersionCreateConfig } from '@shared/models/settings.models'; +import { TranslateService } from '@ngx-translate/core'; +import { DialogService } from '@core/services/dialog.service'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-auto-commit-settings', + templateUrl: './auto-commit-settings.component.html', + styleUrls: ['./auto-commit-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class AutoCommitSettingsComponent extends PageComponent implements OnInit { + + autoCommitSettingsForm: FormGroup; + settings: AutoCommitSettings = null; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private sanitizer: DomSanitizer, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.autoCommitSettingsForm = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.adminService.autoCommitSettingsExists().pipe( + catchError(() => of(false)), + mergeMap((hasAutoCommitSettings) => { + if (hasAutoCommitSettings) { + return this.adminService.getAutoCommitSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(settings), {emitEvent: false}); + }); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.autoCommitSettingsForm.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.autoCommitSettingsForm.get('entityTypes') as FormArray).removeAt(index); + this.autoCommitSettingsForm.markAsDirty(); + } + + public addEnabled(): boolean { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + const config: AutoVersionCreateConfig = { + branch: null, + saveRelations: false, + saveAttributes: false + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + public removeAll() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + entityTypeText(entityTypeControl: AbstractControl): SafeHtml { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: AutoVersionCreateConfig = entityTypeControl.get('config').value; + let message = entityType ? this.translate.instant(entityTypeTranslations.get(entityType).typePlural) : 'Undefined'; + let branchName; + if (config.branch) { + branchName = config.branch; + } else { + branchName = this.translate.instant('version-control.default'); + } + message += ` (${this.translate.instant('version-control.auto-commit-to-branch', {branch: branchName})})`; + return this.sanitizer.bypassSecurityTrustHtml(message); + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + save(): void { + const value: [{entityType: string, config: AutoVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const settings: AutoCommitSettings = {}; + if (value && value.length) { + value.forEach((val) => { + settings[val.entityType] = val.config; + }); + } + this.adminService.saveAutoCommitSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(savedSettings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-auto-commit-settings-title', ), + this.translate.instant('admin.delete-auto-commit-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteAutoCommitSettings().subscribe( + () => { + this.settings = null; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(this.settings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + }); + } + + private prepareEntityTypesFormArray(settings: AutoCommitSettings | null): FormArray { + const entityTypesControls: Array = []; + if (settings) { + for (const entityType of Object.keys(settings)) { + const config = settings[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: AutoVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + branch: [config.branch, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []] + }) + } + ); + return entityTypeControl; + } + + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 3c1dd73117..d4da91028f 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -17,7 +17,7 @@ -->
- version-control.entity-types + version-control.entities-to-export
version-control.no-entity-types + class="tb-prompt">version-control.no-entities-to-export-prompt
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 2ff8c03227..cc5ddcb568 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -41,6 +41,9 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni @Input() entityId: EntityId; + @Input() + entityName: string; + @Input() onClose: (result: VersionCreationResult | null, branch: string | null) => void; @@ -61,7 +64,8 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni ngOnInit(): void { this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], - versionName: [null, [Validators.required]], + versionName: [this.translate.instant('version-control.default-create-entity-version-name', + {entityName: this.entityName}), [Validators.required]], saveRelations: [false, []], saveAttributes: [false, []] }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts index 6036cf35a0..c72cbc0848 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -308,7 +308,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, this.popoverService.hidePopover(trigger); } else { const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, - this.viewContainerRef, EntityVersionRestoreComponent, 'left', true, null, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, { branch: this.branch, versionName: this.versionName, @@ -322,6 +322,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, } } }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; } } } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index e84beb03dc..dd851e9f23 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -15,27 +15,28 @@ limitations under the License. --> -
- +
+

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

 + *ngIf="entityDataInfo && (isLoading$ | async)"> -
+ +
- + {{ 'version-control.load-entity-relations' | translate }} - + {{ 'version-control.load-entity-attributes' | translate }}
-
+
- {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index d4da91028f..0ab5eca4e0 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -66,10 +66,10 @@
- {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index d5a08ee141..c07fa138be 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -65,10 +65,10 @@
- {{ 'version-control.load-entities-relations' | translate }} + {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-entities-attributes' | translate }} + {{ 'version-control.load-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index b5f2fb6937..fa2855a6ed 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -39,10 +39,10 @@ - {{ 'version-control.export-entity-relations' | translate }} + {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-entity-attributes' | translate }} + {{ 'version-control.export-attributes' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index e84beb03dc..0d97823c21 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -27,10 +27,10 @@
- {{ 'version-control.load-entity-relations' | translate }} + {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-entity-attributes' | translate }} + {{ 'version-control.load-attributes' | translate }}
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 40d427df24..2f428f11fe 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3125,8 +3125,8 @@ "version-name": "Version name", "version-name-required": "Version name is required", "author": "Author", - "export-entity-relations": "Export entity relations", - "export-entity-attributes": "Export entity attributes", + "export-relations": "Export relations", + "export-attributes": "Export attributes", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3138,8 +3138,8 @@ "nothing-to-commit": "No changes to commit", "restore-version": "Restore version", "restore-entity-from-version": "Restore entity from version '{{versionName}}'", - "load-entity-relations": "Load entity relations", - "load-entity-attributes": "Load entity attributes", + "load-relations": "Load relations", + "load-attributes": "Load attributes", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3159,8 +3159,6 @@ "add-entity-type": "Add entity type", "remove-all": "Remove all", "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", - "load-entities-relations": "Load entities relations", - "load-entities-attributes": "Load entities attributes", "remove-other-entities": "Remove other entities", "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", From 8382850d1a0c056561e6b071477f5b273a9cdee8 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 1 Jun 2022 16:35:20 +0300 Subject: [PATCH 096/156] UI: Minor improvements --- .../vc/auto-commit-settings.component.html | 6 +++--- .../vc/auto-commit-settings.component.ts | 4 ++-- .../vc/complex-version-create.component.html | 3 ++- .../vc/complex-version-create.component.ts | 4 +++- .../entity-types-version-create.component.html | 6 +++--- .../vc/entity-types-version-create.component.ts | 4 ++-- .../vc/entity-types-version-load.component.html | 6 +++--- .../vc/entity-types-version-load.component.ts | 6 +++--- .../vc/entity-version-create.component.html | 6 +++--- .../vc/entity-version-create.component.ts | 2 +- .../vc/entity-version-restore.component.html | 6 +++--- .../vc/entity-version-restore.component.ts | 4 ++-- .../app/shared/components/popover.component.ts | 12 +++++++----- .../app/shared/components/popover.service.ts | 8 ++------ ui-ngx/src/app/shared/models/vc.models.ts | 17 ++++++++++++----- .../assets/locale/locale.constant-en_US.json | 4 +++- 16 files changed, 54 insertions(+), 44 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index a33d9ea374..e95cfc59fb 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -75,12 +75,12 @@
- - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index 8ac301338e..dc009d49cd 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -97,8 +97,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; const config: AutoVersionCreateConfig = { branch: null, - saveRelations: false, - saveAttributes: false + saveAttributes: true, + saveRelations: false }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html index 2c2f5bf274..4ace5e8849 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -39,13 +39,14 @@ {{ 'version-control.version-name-required' | translate }} - + version-control.default-sync-strategy {{syncStrategyTranslations.get(strategy) | translate}} + diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts index c152e6b447..c5c4124627 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -20,7 +20,7 @@ import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { ComplexVersionCreateRequest, createDefaultEntityTypesVersionCreate, - SyncStrategy, syncStrategyTranslationMap, + SyncStrategy, syncStrategyHintMap, syncStrategyTranslationMap, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -51,6 +51,8 @@ export class ComplexVersionCreateComponent extends PageComponent implements OnIn syncStrategyTranslations = syncStrategyTranslationMap; + syncStrategyHints = syncStrategyHintMap; + resultMessage: string; versionCreateResult: VersionCreationResult = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 0ab5eca4e0..90712727e4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -65,12 +65,12 @@
- - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts index 44cb11dfa1..0d80b71adb 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -183,8 +183,8 @@ export class EntityTypesVersionCreateComponent extends PageComponent implements const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionCreateConfig = { syncStrategy: null, - saveRelations: false, - saveAttributes: false, + saveAttributes: true, + saveRelations: true, allEntities: true, entityIds: [] }; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html index c07fa138be..4c1cd26f84 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -64,12 +64,12 @@
- - {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-attributes' | translate }} + + {{ 'version-control.load-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts index 14586a5ee6..d983920cec 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -164,10 +164,10 @@ export class EntityTypesVersionLoadComponent extends PageComponent implements On public addEntityType() { const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; const config: EntityTypeVersionLoadConfig = { - loadRelations: false, - loadAttributes: false, + loadAttributes: true, + loadRelations: true, removeOtherEntities: false, - findExistingEntityByName: false + findExistingEntityByName: true }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html index 72dc6e7d48..90f338ad0e 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -38,12 +38,12 @@ {{ 'version-control.version-name-required' | translate }} - - {{ 'version-control.export-relations' | translate }} - {{ 'version-control.export-attributes' | translate }} + + {{ 'version-control.export-relations' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index cc5ddcb568..c2c6fa48b5 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -67,7 +67,7 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni versionName: [this.translate.instant('version-control.default-create-entity-version-name', {entityName: this.entityName}), [Validators.required]], saveRelations: [false, []], - saveAttributes: [false, []] + saveAttributes: [true, []] }); } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index 6a39e272e3..8dc81e38e4 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -27,12 +27,12 @@
- - {{ 'version-control.load-relations' | translate }} - {{ 'version-control.load-attributes' | translate }} + + {{ 'version-control.load-relations' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 5e3a63243d..146a8570c2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -70,8 +70,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ - loadRelations: [true, []], - loadAttributes: [true, []] + loadAttributes: [true, []], + loadRelations: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; diff --git a/ui-ngx/src/app/shared/components/popover.component.ts b/ui-ngx/src/app/shared/components/popover.component.ts index c7e498a236..99321974af 100644 --- a/ui-ngx/src/app/shared/components/popover.component.ts +++ b/ui-ngx/src/app/shared/components/popover.component.ts @@ -55,7 +55,7 @@ import { POSITION_MAP, PropertyMapping } from '@shared/components/popover.models'; -import { distinctUntilChanged, takeUntil } from 'rxjs/operators'; +import { distinctUntilChanged, take, takeUntil } from 'rxjs/operators'; import { isNotEmptyStr, onParentScrollOrWindowResize } from '@core/utils'; export type TbPopoverTrigger = 'click' | 'focus' | 'hover' | null; @@ -372,7 +372,7 @@ export class TbPopoverComponent implements OnDestroy, OnInit { } get tbVisible(): boolean { - return this.visible; + return this.visible && this.tbAnimationState === 'active'; } visible = false; @@ -514,10 +514,12 @@ export class TbPopoverComponent implements OnDestroy, OnInit { const el = this.origin.elementRef.nativeElement; this.intersectionObserver.unobserve(el); } - - this.tbVisible = false; - this.tbVisibleChange.next(false); + this.tbAnimationState = 'void'; this.cdr.detectChanges(); + this.tbAnimationDone.pipe(take(1)).subscribe(() => { + this.tbVisible = false; + this.cdr.detectChanges(); + }); } updateByDirective(): void { diff --git a/ui-ngx/src/app/shared/components/popover.service.ts b/ui-ngx/src/app/shared/components/popover.service.ts index 2b64ee8009..713e8d337b 100644 --- a/ui-ngx/src/app/shared/components/popover.service.ts +++ b/ui-ngx/src/app/shared/components/popover.service.ts @@ -80,9 +80,7 @@ export class TbPopoverService { component.tbShowCloseButton = showCloseButton; component.tbVisibleChange.subscribe((visible: boolean) => { if (!visible) { - component.tbAnimationDone.subscribe(() => { - componentRef.destroy(); - }); + componentRef.destroy(); } }); component.tbDestroy.subscribe(() => { @@ -132,9 +130,7 @@ export class TbPopoverService { component.tbVisibleChange.subscribe((visible: boolean) => { if (!visible) { visibleFn(false); - component.tbAnimationDone.subscribe(() => { - componentRef.destroy(); - }); + componentRef.destroy(); } }); component.tbDestroy.subscribe(() => { diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index e3a7be5c54..ee718c3aec 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -64,6 +64,13 @@ export const syncStrategyTranslationMap = new Map( ] ); +export const syncStrategyHintMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge-hint'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite-hint'] + ] +); + export interface EntityTypeVersionCreateConfig extends VersionCreateConfig { syncStrategy: SyncStrategy; entityIds: string[]; @@ -81,8 +88,8 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: for (const entityType of exportableEntityTypes) { res[entityType] = { syncStrategy: null, - saveRelations: false, - saveAttributes: false, + saveAttributes: true, + saveRelations: true, allEntities: true, entityIds: [] }; @@ -126,10 +133,10 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; for (const entityType of exportableEntityTypes) { res[entityType] = { - loadRelations: false, - loadAttributes: false, + loadAttributes: true, + loadRelations: true, removeOtherEntities: false, - findExistingEntityByName: false + findExistingEntityByName: true }; } return res; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2bc8d6fdbe..ad9ba856ab 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3168,7 +3168,9 @@ "deleted": "{{deleted}} deleted", "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm.", "auto-commit-to-branch": "auto-commit to {{ branch }} branch", - "default-create-entity-version-name": "{{entityName}} update" + "default-create-entity-version-name": "{{entityName}} update", + "sync-strategy-merge-hint": "Sync strategy merge hint", + "sync-strategy-overwrite-hint": "Sync strategy overwrite hint" }, "widget": { "widget-library": "Widgets Library", From 54e388433479d7e8259bb05d5151fe85196a6c3c Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 1 Jun 2022 16:55:38 +0300 Subject: [PATCH 097/156] Version load error and 'exportCredentials' setting support --- .../EntitiesVersionControlController.java | 3 +- .../impl/BaseEntityExportService.java | 4 +- .../exporting/impl/DeviceExportService.java | 7 +- .../impl/RuleChainExportService.java | 3 +- .../ie/importing/impl/AssetImportService.java | 3 +- .../impl/BaseEntityImportService.java | 12 +- .../importing/impl/CustomerImportService.java | 3 +- .../impl/DashboardImportService.java | 2 +- .../importing/impl/DeviceImportService.java | 5 +- .../impl/DeviceProfileImportService.java | 3 +- .../impl/ImportServiceException.java | 20 ++ .../impl/MissingEntityException.java | 30 +++ .../impl/RuleChainImportService.java | 2 +- .../DefaultEntitiesVersionControlService.java | 251 ++++++++++-------- .../DefaultGitVersionControlQueueService.java | 6 +- .../vc/EntitiesVersionControlService.java | 5 +- .../service/sync/vc/LoadEntityException.java | 32 +++ .../common/data/sync/ie/EntityExportData.java | 6 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../common/data/sync/vc/EntityDataInfo.java | 1 + .../common/data/sync/vc/EntityLoadError.java | 42 +++ .../data/sync/vc/EntityTypeLoadResult.java | 33 +++ .../data/sync/vc/VersionLoadResult.java | 29 +- .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 26 files changed, 368 insertions(+), 138 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 58c2659c4a..d55fe3dec0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -296,7 +297,7 @@ public class EntitiesVersionControlController extends BaseController { " }\n" + "}\n```") @PostMapping("/entity") - public DeferredResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + public DeferredResult loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { SecurityUser user = getCurrentUser(); try { return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 80850cb462..9e14531e48 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -30,11 +30,11 @@ public abstract class BaseEntityExportService exportData, IdProvider idProvider) { + protected Asset prepareAndSave(TenantId tenantId, Asset asset, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { return assetService.saveAsset(asset); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index caa1e20ad7..ea938c89d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -64,7 +64,8 @@ import java.util.stream.Collectors; @Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { - @Autowired @Lazy + @Autowired + @Lazy private ExportableEntitiesService exportableEntitiesService; @Autowired private RelationService relationService; @@ -94,7 +95,7 @@ public abstract class BaseEntityImportService importResult = new EntityImportResult<>(); importResult.setSavedEntity(savedEntity); @@ -108,7 +109,7 @@ public abstract class BaseEntityImportService importResult, D exportData, @@ -210,7 +211,8 @@ public abstract class BaseEntityImportService() { @Override - public void onSuccess(@Nullable Void unused) {} + public void onSuccess(@Nullable Void unused) { + } @Override public void onFailure(Throwable thr) { @@ -246,7 +248,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) - .orElseThrow(() -> new IllegalArgumentException("Cannot find " + externalId.getEntityType() + " by external id " + externalId)); + .orElseThrow(() -> new MissingEntityException(externalId)); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index aec96b2e47..2b2e2d1ddd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,7 +42,7 @@ public class CustomerImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Customer prepareAndSave(TenantId tenantId, Customer customer, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { if (customer.isPublic()) { return customerService.findOrCreatePublicCustomer(tenantId); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 16ebdd580e..a67a536ded 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -64,7 +64,7 @@ public class DashboardImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected Dashboard prepareAndSave(TenantId tenantId, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { JsonNode configuration = dashboard.getConfiguration(); String newConfigurationJson = RegexUtils.replace(configuration.toString(), RegexUtils.UUID_PATTERN, uuid -> { return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index eb123a63af..9a3730bba9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -41,11 +42,11 @@ public class DeviceImportService extends BaseEntityImportService exportData, IdProvider idProvider) { + protected DeviceProfile prepareAndSave(TenantId tenantId, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider, EntityImportSettings importSettings) { deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); deviceProfile.setFirmwareId(idProvider.getInternalId(deviceProfile.getFirmwareId())); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java new file mode 100644 index 0000000000..1e869429d8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +public class ImportServiceException extends RuntimeException{ + private static final long serialVersionUID = -4932715239522125041L; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java new file mode 100644 index 0000000000..a0a961bfef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; + +public class MissingEntityException extends ImportServiceException { + + private static final long serialVersionUID = 3669135386955906022L; + @Getter + private final EntityId entityId; + + public MissingEntityException(EntityId entityId) { + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 1f1f15a74e..cd29be99d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -62,7 +62,7 @@ public class RuleChainImportService extends BaseEntityImportService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 735dc33a24..8aa4915fa8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -22,8 +22,10 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ObjectUtils; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -45,10 +47,12 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityLoadError; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; @@ -64,12 +68,14 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; @@ -83,8 +89,10 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -172,6 +180,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -203,123 +212,157 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) @Override - public ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + public ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; VersionLoadConfig config = versionLoadRequest.getConfig(); ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); - return Futures.transform(future, entityData -> { - EntityImportResult importResult = transactionTemplate.execute(status -> { - try { - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(false) - .build(), true, true); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - return List.of(VersionLoadResult.builder() - .entityType(importResult.getEntityType()) - .created(importResult.getOldEntity() == null ? 1 : 0) - .updated(importResult.getOldEntity() != null ? 1 : 0) - .deleted(0) - .build()); - }, executor); + return Futures.transform(future, entityData -> doInTemplate(status -> loadSingleEntity(user, config, entityData)), executor); } case ENTITY_TYPE: { EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; - return executor.submit(() -> transactionTemplate.execute(status -> { - Map results = new HashMap<>(); - Map> importedEntities = new HashMap<>(); - List saveReferencesCallbacks = new ArrayList<>(); - List sendEventsCallbacks = new ArrayList<>(); - - versionLoadRequest.getEntityTypes().keySet().stream() - .sorted(exportImportService.getEntityTypeComparatorForImport()) - .forEach(entityType -> { - EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); + return executor.submit(() -> doInTemplate(status -> loadMultipleEntities(user, versionLoadRequest))); + } + default: + throw new IllegalArgumentException("Unsupported version load request"); + } + } - try { - int limit = 100; - int offset = 0; - List entityDataList; - do { - entityDataList = gitServiceQueue.getEntities(user.getTenantId(), request.getVersionId(), entityType, offset, limit).get(); - for (EntityExportData entityData : entityDataList) { - EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .saveAttributes(config.isLoadAttributes()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - offset += limit; - importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) - .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); - } while (entityDataList.size() == limit); - } catch (Exception e) { - throw new RuntimeException(e); - } - results.put(entityType, VersionLoadResult.builder() - .entityType(entityType) - .created(created.get()) - .updated(updated.get()) - .build()); - }); + private VersionLoadResult doInTemplate(TransactionCallback result) { + try { + return transactionTemplate.execute(result); + } catch (LoadEntityException e) { + return onError(e.getData(), e.getCause()); + } + } - versionLoadRequest.getEntityTypes().keySet().stream() - .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) - .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) - .forEach(entityType -> { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - - sendEventsCallbacks.add(() -> { - entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), - entity, null, ActionType.DELETED, null, user); - }); - VersionLoadResult result = results.get(entityType); - result.setDeleted(result.getDeleted() + 1); - } - }); - }); + private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) { + try { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, + EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) + .findExistingByName(false) + .build(), true, true); + return VersionLoadResult.success(EntityTypeLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + } - for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { - try { - saveReferencesCallback.run(); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } + private VersionLoadResult loadMultipleEntities(SecurityUser user, EntityTypeVersionLoadRequest versionLoadRequest) { + Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); + List saveReferencesCallbacks = new ArrayList<>(); + List sendEventsCallbacks = new ArrayList<>(); + + List entityTypes = versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + for (EntityType entityType : entityTypes) { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); + + int limit = 100; + int offset = 0; + List entityDataList; + do { + try { + entityDataList = gitServiceQueue.getEntities(user.getTenantId(), versionLoadRequest.getVersionId(), entityType, offset, limit).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (EntityExportData entityData : entityDataList) { + try { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); } - for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { - try { - sendEventsCallback.run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getExternalId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + results.put(entityType, EntityTypeLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); + } + + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); + EntityTypeLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); } - } - return new ArrayList<>(results.values()); - })); + }); + }); + + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { + try { + saveReferencesCallback.run(); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + } + for (ThrowingRunnable sendEventsCallback : sendEventsCallbacks) { + try { + sendEventsCallback.run(); + } catch (Exception e) { + log.error("Failed to send events for entity", e); + } + } + return VersionLoadResult.success(new ArrayList<>(results.values())); + } + + private VersionLoadResult onError(EntityExportData entityData, Throwable e) { + return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e)); + } + + private Optional analyze(Throwable e, EntityExportData entityData) { + if (e == null) { + return Optional.empty(); + } else { + if (e instanceof DeviceCredentialsValidationException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId()))); + } else if (e instanceof MissingEntityException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId()))); + } else { + return analyze(e.getCause(), entityData); } - default: - throw new IllegalArgumentException("Unsupported version load request"); } } @@ -347,7 +390,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), - entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null), MoreExecutors.directExecutor()); + entity -> new EntityDataInfo(entity.getRelations() != null, entity.getAttributes() != null, false), MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index e168312206..759bc83acf 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -108,7 +108,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { SettableFuture future = SettableFuture.create(); - String path = getRelativePath(entityData.getEntityType(), getExternalId(entityData.getEntity())); + String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); registerAndSend(commit, builder -> builder.setCommitRequest( @@ -120,10 +120,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return future; } - private EntityId getExternalId(ExportableEntity entity) { - return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); - } - @Override public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { SettableFuture future = SettableFuture.create(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 3b38182075..0a0f56eac6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -23,11 +23,12 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; -import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; @@ -49,7 +50,7 @@ public interface EntitiesVersionControlService { ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - ListenableFuture> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java new file mode 100644 index 0000000000..a1b036d37b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@SuppressWarnings("rawtypes") +public class LoadEntityException extends RuntimeException { + + private static final long serialVersionUID = -1749719992370409504L; + @Getter + private final EntityExportData data; + + public LoadEntityException(EntityExportData data, Throwable cause) { + super(cause); + this.data = data; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 1109d9ef09..d715ec9611 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data.sync.ie; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; @@ -75,4 +76,9 @@ public class EntityExportData> { return this; } + @JsonIgnore + public EntityId getExternalId() { + return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0800a1f7c1..1a625640e0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityExportSettings { private boolean exportRelations; private boolean exportAttributes; + private boolean exportCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 564b0134e3..9b95b8eca2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -28,4 +28,5 @@ public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; private boolean saveAttributes; + private boolean saveCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java index ab548b1cfc..2529ec8af6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -25,4 +25,5 @@ import lombok.NoArgsConstructor; public class EntityDataInfo { boolean hasRelations; boolean hasAttributes; + boolean hasCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java new file mode 100644 index 0000000000..900a05e470 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EntityLoadError { + + private String type; + private EntityId source; + private EntityId target; + + public static EntityLoadError credentialsError(EntityId sourceId) { + return EntityLoadError.builder().type("DEVICE_CREDENTIALS_CONFLICT").source(sourceId).build(); + } + + public static EntityLoadError referenceEntityError(EntityId sourceId, EntityId targetId) { + return EntityLoadError.builder().type("MISSING_REFERENCED_ENTITY").source(sourceId).target(targetId).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java new file mode 100644 index 0000000000..cc0a8a642a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityTypeLoadResult { + private EntityType entityType; + private int created; + private int updated; + private int deleted; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index 0039546fb0..736de26079 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -15,19 +15,30 @@ */ package org.thingsboard.server.common.data.sync.vc; -import lombok.AllArgsConstructor; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; -import org.thingsboard.server.common.data.EntityType; + +import java.util.List; @Data -@AllArgsConstructor -@NoArgsConstructor @Builder +@JsonInclude(JsonInclude.Include.NON_NULL) public class VersionLoadResult { - private EntityType entityType; - private int created; - private int updated; - private int deleted; + + private List result; + private EntityLoadError error; + + public static VersionLoadResult success(List result) { + return VersionLoadResult.builder().result(result).build(); + } + + public static VersionLoadResult success(EntityTypeLoadResult result) { + return VersionLoadResult.builder().result(List.of(result)).build(); + } + + public static VersionLoadResult error(EntityLoadError error) { + return VersionLoadResult.builder().error(error).build(); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 4426717fe9..86154bdfa0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -25,4 +25,5 @@ public class VersionCreateConfig implements Serializable { private boolean saveRelations; private boolean saveAttributes; + private boolean saveCredentials; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 2d68a2b7e3..a27cf06538 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -22,5 +22,6 @@ public class VersionLoadConfig { private boolean loadRelations; private boolean loadAttributes; + private boolean loadCredentials; } From 14023e4cc9fc1a957d8c8a9f81e06d14ea19aed0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 1 Jun 2022 18:02:52 +0300 Subject: [PATCH 098/156] UI: Implement rule chain page version control. --- .../vc/entity-version-create.component.ts | 44 +++++++----- .../vc/entity-versions-table.component.html | 7 +- .../vc/entity-versions-table.component.ts | 9 ++- .../vc/version-control.component.html | 2 + .../vc/version-control.component.ts | 7 ++ .../rulechain/rulechain-page.component.html | 11 +++ .../rulechain/rulechain-page.component.scss | 15 ++++ .../rulechain/rulechain-page.component.ts | 68 +++++++++++++++++-- .../shared/components/breadcrumb.component.ts | 12 +++- 9 files changed, 148 insertions(+), 27 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index c2c6fa48b5..108d4aa081 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -27,6 +27,7 @@ import { AppState } from '@core/core.state'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { EntityId } from '@shared/models/id/entity-id'; import { TranslateService } from '@ngx-translate/core'; +import { Observable, of } from 'rxjs'; @Component({ selector: 'tb-entity-version-create', @@ -50,6 +51,9 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni @Input() onContentUpdated: () => void; + @Input() + onBeforeCreateVersion: () => Observable; + createVersionFormGroup: FormGroup; resultMessage: string; @@ -78,25 +82,29 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni } export(): void { - const request: SingleEntityVersionCreateRequest = { - entityId: this.entityId, - branch: this.createVersionFormGroup.get('branch').value, - versionName: this.createVersionFormGroup.get('versionName').value, - config: { - saveRelations: this.createVersionFormGroup.get('saveRelations').value, - saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, - }, - type: VersionCreateRequestType.SINGLE_ENTITY - }; - this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { - if (!result.added && !result.modified) { - this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); - if (this.onContentUpdated) { - this.onContentUpdated(); + const before = this.onBeforeCreateVersion ? this.onBeforeCreateVersion() : of(null); + before.subscribe(() => { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + config: { + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + if (this.onContentUpdated) { + this.onContentUpdated(); + } + } else if (this.onClose) { + this.onClose(result, request.branch); } - } else if (this.onClose) { - this.onClose(result, request.branch); - } + }); }); } } + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index d2dbc119bf..79da474200 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -16,7 +16,7 @@ -->
-
+
@@ -45,6 +45,11 @@ update {{'version-control.create-entities-version' | translate }} +
+ + {{ 'version-control.export-credentials' | translate }} + {{ 'version-control.export-attributes' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts index dc009d49cd..fd100f6247 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -39,6 +39,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit autoCommitSettingsForm: FormGroup; settings: AutoCommitSettings = null; + entityTypes = EntityType; + constructor(protected store: Store, private adminService: AdminService, private dialogService: DialogService, @@ -98,7 +100,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit const config: AutoVersionCreateConfig = { branch: null, saveAttributes: true, - saveRelations: false + saveRelations: false, + saveCredentials: true }; const allowed = this.allowedEntityTypes(); let entityType: EntityType = null; @@ -199,7 +202,8 @@ export class AutoCommitSettingsComponent extends PageComponent implements OnInit config: this.fb.group({ branch: [config.branch, []], saveRelations: [config.saveRelations, []], - saveAttributes: [config.saveAttributes, []] + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []] }) } ); diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html index 4ace5e8849..9e15076788 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -70,7 +70,7 @@
-
{{ resultMessage }}
+
-
-
+
+
{{ 'version-control.no-entities-restored' | translate }}
-
{{ versionLoadResultMessage(versionLoadResult) }}
+
+
{{ entityTypeLoadResultMessage(entityTypeLoadResult) }}
-
{{ resultMessage }}
+
{{ resultMessage }}
- -
+
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + + +
+
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts index 146a8570c2..c31030f74b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -30,11 +30,12 @@ import { EntityId } from '@shared/models/id/entity-id'; import { TranslateService } from '@ngx-translate/core'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { delay } from 'rxjs/operators'; +import { SafeHtml } from '@angular/platform-browser'; @Component({ selector: 'tb-entity-version-restore', templateUrl: './entity-version-restore.component.html', - styleUrls: [] + styleUrls: ['./version-control.scss'] }) export class EntityVersionRestoreComponent extends PageComponent implements OnInit { @@ -51,7 +52,7 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn externalEntityId: EntityId; @Input() - onClose: (result: Array | null) => void; + onClose: (result: VersionLoadResult | null) => void; @Input() popoverComponent: TbPopoverComponent; @@ -60,6 +61,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn restoreFormGroup: FormGroup; + errorMessage: SafeHtml; + constructor(protected store: Store, private entitiesVersionControlService: EntitiesVersionControlService, private cd: ChangeDetectorRef, @@ -71,7 +74,8 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn ngOnInit(): void { this.restoreFormGroup = this.fb.group({ loadAttributes: [true, []], - loadRelations: [true, []] + loadRelations: [true, []], + loadCredentials: [true, []] }); this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { this.entityDataInfo = data; @@ -95,13 +99,22 @@ export class EntityVersionRestoreComponent extends PageComponent implements OnIn externalEntityId: this.externalEntityId, config: { loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, - loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false + loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false }, type: VersionLoadRequestType.SINGLE_ENTITY }; this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { - if (this.onClose) { - this.onClose(result); + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else { + if (this.onClose) { + this.onClose(result); + } } }); } diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index 79da474200..b6accc40dc 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -15,8 +15,8 @@ limitations under the License. --> -
-
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss index ec5cb02256..3256c3c956 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -20,6 +20,13 @@ height: 100%; display: block; .tb-entity-table { + + &.tb-popover-mode { + position: relative; + width: 800px; + height: 600px; + } + .tb-entity-table-content { width: 100%; height: 100%; diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts index 38b0a33da4..d776939aaf 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -49,6 +49,7 @@ import { EntityVersionRestoreComponent } from '@home/components/vc/entity-versio import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-entity-versions-table', @@ -63,7 +64,7 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni singleEntityMode = false; @Input() - popoverMode = false; + popoverComponent: TbPopoverComponent; @Input() onBeforeCreateVersion: () => Observable; @@ -211,14 +212,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } - }, - onContentUpdated: () => { - createVersionPopover.updatePosition(); - setTimeout(() => { - createVersionPopover.updatePosition(); - }); } }, {}, {}, {}, false); + createVersionPopover.tbComponentRef.instance.popoverComponent = createVersionPopover; } } @@ -243,14 +239,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni this.updateData(); } } - }, - onContentUpdated: () => { - complexCreateVersionPopover.updatePosition(); - setTimeout(() => { - complexCreateVersionPopover.updatePosition(); - }); } }, {}, {}, {}, false); + complexCreateVersionPopover.tbComponentRef.instance.popoverComponent = complexCreateVersionPopover; } } @@ -293,9 +284,9 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni versionName: entityVersion.name, versionId: entityVersion.id, externalEntityId: this.externalEntityIdValue, - onClose: (result: Array | null) => { + onClose: (result: VersionLoadResult | null) => { restoreVersionPopover.hide(); - if (result && result.length) { + if (result && !result.error && result.result.length) { this.versionRestored.emit(); } } @@ -318,10 +309,11 @@ export class EntityVersionsTableComponent extends PageComponent implements OnIni branch: this.branch, versionName: entityVersion.name, versionId: entityVersion.id, - onClose: (result: Array | null) => { + onClose: (result: VersionLoadResult | null) => { restoreEntitiesVersionPopover.hide(); } }, {}, {}, {}, false); + restoreEntitiesVersionPopover.tbComponentRef.instance.popoverComponent = restoreEntitiesVersionPopover; } } diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html index bdab6c061b..3e8bd271f3 100644 --- a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -16,7 +16,7 @@ -->
- +
admin.repository-settings diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts index c73cf0e632..1120185db9 100644 --- a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { Component, Input, OnInit } from '@angular/core'; +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; import { PageComponent } from '@shared/components/page.component'; import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; import { select, Store } from '@ngrx/store'; @@ -33,6 +33,7 @@ import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; import { selectHasRepository } from '@core/auth/auth.selectors'; import { catchError, mergeMap, take } from 'rxjs/operators'; import { of } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; @Component({ selector: 'tb-repository-settings', @@ -44,6 +45,9 @@ export class RepositorySettingsComponent extends PageComponent implements OnInit @Input() detailsMode = false; + @Input() + popoverComponent: TbPopoverComponent; + repositorySettingsForm: FormGroup; settings: RepositorySettings = null; @@ -61,6 +65,7 @@ export class RepositorySettingsComponent extends PageComponent implements OnInit private adminService: AdminService, private dialogService: DialogService, private translate: TranslateService, + private cd: ChangeDetectorRef, public fb: FormBuilder) { super(store); } diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html index 0ede98554d..eaf80fd664 100644 --- a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -16,11 +16,12 @@ --> { this.reloadRuleChain(); }); diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts index ee718c3aec..0c6a667b88 100644 --- a/ui-ngx/src/app/shared/models/vc.models.ts +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -33,6 +33,7 @@ export const exportableEntityTypes: Array = [ export interface VersionCreateConfig { saveRelations: boolean; saveAttributes: boolean; + saveCredentials: boolean; } export enum VersionCreateRequestType { @@ -90,6 +91,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: syncStrategy: null, saveAttributes: true, saveRelations: true, + saveCredentials: true, allEntities: true, entityIds: [] }; @@ -100,6 +102,7 @@ export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: export interface VersionLoadConfig { loadRelations: boolean; loadAttributes: boolean; + loadCredentials: boolean; } export enum VersionLoadRequestType { @@ -135,6 +138,7 @@ export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: En res[entityType] = { loadAttributes: true, loadRelations: true, + loadCredentials: true, removeOtherEntities: false, findExistingEntityByName: true }; @@ -161,13 +165,36 @@ export interface VersionCreationResult { removed: number; } -export interface VersionLoadResult { +export interface EntityTypeLoadResult { entityType: EntityType; created: number; updated: number; deleted: number; } +export enum EntityLoadErrorType { + DEVICE_CREDENTIALS_CONFLICT = 'DEVICE_CREDENTIALS_CONFLICT', + MISSING_REFERENCED_ENTITY = 'MISSING_REFERENCED_ENTITY' +} + +export const entityLoadErrorTranslationMap = new Map( + [ + [EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT, 'version-control.device-credentials-conflict'], + [EntityLoadErrorType.MISSING_REFERENCED_ENTITY, 'version-control.missing-referenced-entity'] + ] +); + +export interface EntityLoadError { + type: EntityLoadErrorType; + source: EntityId; + target: EntityId; +} + +export interface VersionLoadResult { + result: Array; + error: EntityLoadError; +} + export interface EntityExportData> { entity: E; entityType: EntityType; @@ -195,4 +222,5 @@ export function entityExportDataToJsonString(data: EntityExportData): strin export interface EntityDataInfo { hasRelations: boolean; hasAttributes: boolean; + hasCredentials: boolean; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index ad9ba856ab..8a4d8ff4db 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3127,6 +3127,7 @@ "author": "Author", "export-relations": "Export relations", "export-attributes": "Export attributes", + "export-credentials": "Export credentials", "entity-versions": "Entity versions", "versions": "Versions", "created-time": "Created time", @@ -3140,6 +3141,7 @@ "restore-entity-from-version": "Restore entity from version '{{versionName}}'", "load-relations": "Load relations", "load-attributes": "Load attributes", + "load-credentials": "Load credentials", "show-version-diff": "Show version diff", "diff-entity-with-version": "Diff with entity version '{{versionName}}'", "previous-difference": "Previous Difference", @@ -3158,7 +3160,7 @@ "no-entities-to-restore-prompt": "Please specify entities to restore", "add-entity-type": "Add entity type", "remove-all": "Remove all", - "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.\n{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.\n{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.
{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.
{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", "remove-other-entities": "Remove other entities", "find-existing-entity-by-name": "Find existing entity by name", "restore-entities-from-version": "Restore entities from version '{{versionName}}'", @@ -3170,7 +3172,9 @@ "auto-commit-to-branch": "auto-commit to {{ branch }} branch", "default-create-entity-version-name": "{{entityName}} update", "sync-strategy-merge-hint": "Sync strategy merge hint", - "sync-strategy-overwrite-hint": "Sync strategy overwrite hint" + "sync-strategy-overwrite-hint": "Sync strategy overwrite hint", + "device-credentials-conflict": "Failed to load the device with external id {{entityId}}
due to the same credentials are already present in the database for another device.
Please consider disabling the load credentials setting in the restore form.", + "missing-referenced-entity": "Failed to load the {{sourceEntityTypeName}} with external id {{sourceEntityId}}
because it references missing {{targetEntityTypeName}} with id {{targetEntityId}}." }, "widget": { "widget-library": "Widgets Library", From da224c9651f6ac14993776704b82aed6a2b319ee Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 2 Jun 2022 14:10:56 +0300 Subject: [PATCH 103/156] UI: Implement dashboard version control --- .../dashboard-page.component.html | 7 +++ .../dashboard-page.component.ts | 60 ++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index dd7212cf5b..553971005f 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -81,6 +81,13 @@ (click)="isFullscreen = !isFullscreen"> {{ isFullscreen ? 'fullscreen_exit' : 'fullscreen' }} +