diff --git a/application/pom.xml b/application/pom.xml index 2d6dda0bbe..6d1e009d00 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -69,6 +69,10 @@ org.thingsboard.common cluster-api + + org.thingsboard.common + version-control + org.thingsboard.rule-engine rule-engine-components diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index 7584d2f732..69df2afdc9 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -14,6 +14,40 @@ -- limitations under the License. -- +ALTER TABLE device + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE device_profile + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE asset + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_chain + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_node + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE dashboard + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE customer + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE widgets_bundle + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE entity_view + ADD COLUMN IF NOT EXISTS external_id UUID; + +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_node_external_id ON rule_node(rule_chain_id, external_id); +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_node_type ON rule_node(type); + +ALTER TABLE admin_settings + ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; + CREATE TABLE IF NOT EXISTS queue ( id uuid NOT NULL CONSTRAINT queue_pkey PRIMARY KEY, created_time bigint NOT NULL, @@ -35,3 +69,4 @@ CREATE TABLE IF NOT EXISTS user_auth_settings ( user_id uuid UNIQUE NOT NULL CONSTRAINT fk_user_auth_settings_user_id REFERENCES tb_user(id), two_fa_settings varchar ); + diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 500ad60524..733cf75e64 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -48,7 +48,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 1415ff4ece..86675a79a8 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.config; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; +import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -41,6 +43,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +@Slf4j @Component public class RateLimitProcessingFilter extends GenericFilterBean { @@ -58,7 +61,13 @@ public class RateLimitProcessingFilter extends GenericFilterBean { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultProfileConfiguration(); + var profile = tenantProfileCache.get(user.getTenantId()); + if (profile == null) { + log.debug("[{}] Failed to lookup tenant profile", user.getTenantId()); + errorResponseHandler.handle(new BadCredentialsException("Failed to lookup tenant profile"), (HttpServletResponse) response); + return; + } + var profileConfiguration = profile.getDefaultProfileConfiguration(); if (!checkRateLimits(user.getTenantId(), profileConfiguration.getTenantServerRestLimitsConfiguration(), perTenantLimits, response)) { return; } diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index e5934cbf26..54cbdd9b0f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -16,16 +16,16 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; @@ -34,14 +34,18 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.update.UpdateService; -import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -60,6 +64,12 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Autowired + private EntitiesVersionControlService versionControlService; + + @Autowired + private TbAutoCommitSettingsService autoCommitSettingsService; + @Autowired private UpdateService updateService; @@ -96,6 +106,7 @@ public class AdminController extends BaseController { @RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + adminSettings.setTenantId(getTenantId()); adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings)); if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); @@ -180,6 +191,137 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get repository settings (getRepositorySettings)", + notes = "Get the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/repositorySettings") + @ResponseBody + public RepositorySettings getRepositorySettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check repository settings exists (repositorySettingsExists)", + notes = "Check whether the repository settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/repositorySettings/exists") + @ResponseBody + public Boolean repositorySettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the repository settings (saveRepositorySettings)", + notes = "Creates or Updates the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/repositorySettings") + public DeferredResult saveRepositorySettings(@RequestBody RepositorySettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); + ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); + return wrapFuture(Futures.transform(future, savedSettings -> { + savedSettings.setPassword(null); + savedSettings.setPrivateKey(null); + savedSettings.setPrivateKeyPassword(null); + return savedSettings; + }, MoreExecutors.directExecutor())); + } + + @ApiOperation(value = "Delete repository settings (deleteRepositorySettings)", + notes = "Deletes the repository settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public DeferredResult deleteRepositorySettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.DELETE); + return wrapFuture(versionControlService.deleteVersionControlSettings(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiOperation(value = "Check repository access (checkRepositoryAccess)", + notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings/checkAccess", method = RequestMethod.POST) + public DeferredResult checkRepositoryAccess( + @ApiParam(value = "A JSON value representing the Repository Settings.") + @RequestBody RepositorySettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + settings = checkNotNull(settings); + return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", + notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/autoCommitSettings") + @ResponseBody + public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return checkNotNull(autoCommitSettingsService.get(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", + notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/autoCommitSettings/exists") + @ResponseBody + public Boolean autoCommitSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return autoCommitSettingsService.get(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", + notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/autoCommitSettings") + public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); + return autoCommitSettingsService.save(getTenantId(), settings); + } + + @ApiOperation(value = "Delete auto commit settings (deleteAutoCommitSettings)", + notes = "Deletes the auto commit settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/autoCommitSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.DELETE); + autoCommitSettingsService.delete(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index b0ed1ed01d..939e9da8a0 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -226,11 +226,7 @@ public class AlarmController extends BaseController { TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); try { - if (getCurrentUser().isCustomerUser()) { - return checkNotNull(alarmService.findCustomerAlarms(getCurrentUser().getTenantId(), getCurrentUser().getCustomerId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); - } else { - return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); - } + return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 4718feb774..2a3268146f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -51,9 +51,9 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.asset.TbAssetService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 050745844b..587354c0c4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -18,6 +18,10 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -29,6 +33,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.MethodArgumentNotValidException; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -138,6 +143,7 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -281,6 +287,9 @@ public abstract class BaseController { @Autowired protected QueueService queueService; + @Autowired + protected EntitiesVersionControlService vcService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -989,4 +998,20 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + protected DeferredResult wrapFuture(ListenableFuture future) { + final DeferredResult deferredResult = new DeferredResult<>(); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(T result) { + deferredResult.setResult(result); + } + + @Override + public void onFailure(Throwable t) { + deferredResult.setErrorResult(t); + } + }, MoreExecutors.directExecutor()); + return deferredResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index a26493000a..63be9a437a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -135,6 +135,8 @@ public class ControllerConstants { protected static final String EDGE_ASSIGN_ASYNC_FIRST_STEP_DESCRIPTION = "Assignment works in async way - first, notification event pushed to edge service queue on platform. "; protected static final String EDGE_ASSIGN_RECEIVE_STEP_DESCRIPTION = "(Edge will receive this instantly, if it's currently connected, or once it's going to be connected to platform). "; + protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name."; + protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n"; protected static final String MARKDOWN_CODE_BLOCK_END = "\n```"; protected static final String EVENT_ERROR_FILTER_OBJ = MARKDOWN_CODE_BLOCK_START + diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 47acdc64a4..d0f40a476b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -631,7 +631,7 @@ public class DashboardController extends BaseController { DashboardId dashboardId = new DashboardId(toUUID(strDashboardId)); checkDashboardId(dashboardId, Operation.READ); - return tbDashboardService.asignDashboardToEdge(dashboardId, edge, getCurrentUser()); + return tbDashboardService.assignDashboardToEdge(dashboardId, edge, getCurrentUser()); } @ApiOperation(value = "Unassign dashboard from edge (unassignDashboardFromEdge)", diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 793ea64336..2881787d95 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -66,9 +66,9 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.device.TbDeviceService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 5d9737db23..4dcffa605e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -52,8 +52,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java new file mode 100644 index 0000000000..701719af58 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -0,0 +1,368 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; + +@RestController +@TbCoreComponent +@RequestMapping("/api/entities/vc") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") +@RequiredArgsConstructor +public class EntitiesVersionControlController extends BaseController { + + private final EntitiesVersionControlService versionControlService; + + + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"saveRelations\": true\n" + + " }\n" + + "}\n```" + NEW_LINE + + "COMPLEX:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"COMPLEX\",\n" + + "\n" + + " \"versionName\": \"Devices and profiles: release 2\",\n" + + " \"branch\": \"master\",\n" + + "\n" + + " \"syncStrategy\": \"OVERWRITE\",\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"syncStrategy\": null,\n" + + " \"allEntities\": true,\n" + + " \"saveRelations\": true\n" + + " },\n" + + " \"DEVICE_PROFILE\": {\n" + + " \"syncStrategy\": \"MERGE\",\n" + + " \"allEntities\": false,\n" + + " \"entityIds\": [\n" + + " \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " ],\n" + + " \"saveRelations\": true\n" + + " }\n" + + " }\n" + + "}\n```") + @PostMapping("/version") + public DeferredResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); + return wrapFuture(versionControlService.saveEntitiesVersion(user, request)); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profile 1 version 1.0\"\n" + + " }\n" + + "]\n```") + @GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) + public DeferredResult> listEntityVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink)); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Device profiles from dev\"\n" + + " }\n" + + "]\n```") + @GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) + public DeferredResult> listEntityTypeVersions(@PathVariable String branch, + @PathVariable EntityType entityType, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink)); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" + + " \"name\": \"Device 1 removed\"\n" + + " },\n" + + " {\n" + + " \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \"name\": \"Device profiles added\"\n" + + " },\n" + + " {\n" + + " \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" + + " \"name\": \"Devices added\"\n" + + " }\n" + + "]\n```") + @GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) + public DeferredResult> listVersions(@PathVariable String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink)); + } catch (Exception e) { + throw handleException(e); + } + } + + + @GetMapping("/entity/{branch}/{entityType}/{versionId}") + public DeferredResult> listEntitiesAtVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable String versionId) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/entity/{branch}/{versionId}") + public DeferredResult> listAllEntitiesAtVersion(@PathVariable String branch, + @PathVariable String versionId) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}") + public DeferredResult getEntityDataInfo(@PathVariable String versionId, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } + + @GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") + public DeferredResult compareEntityDataToVersion(@PathVariable String branch, + @PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid, + @RequestParam String versionId) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId)); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "", notes = "" + + "SINGLE_ENTITY:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + " \n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \n" + + " \"externalEntityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false\n" + + " }\n" + + "}\n```" + NEW_LINE + + "ENTITY_TYPE:" + NEW_LINE + + "```\n{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"loadRelations\": false,\n" + + " \"findExistingEntityByName\": false,\n" + + " \"removeOtherEntities\": true\n" + + " }\n" + + " }\n" + + "}\n```") + @PostMapping("/entity") + public DeferredResult loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + try { + accessControlService.checkPermission(user, Resource.VERSION_CONTROL, Operation.READ); + return wrapFuture(versionControlService.loadEntitiesVersion(user, request)); + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiOperation(value = "", notes = "" + + "```\n[\n" + + " {\n" + + " \"name\": \"master\",\n" + + " \"default\": true\n" + + " },\n" + + " {\n" + + " \"name\": \"dev\",\n" + + " \"default\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"dev-2\",\n" + + " \"default\": false\n" + + " }\n" + + "]\n\n```") + @GetMapping("/branches") + public DeferredResult> listBranches() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + final TenantId tenantId = getTenantId(); + ListenableFuture> branches = versionControlService.listBranches(tenantId); + return wrapFuture(Futures.transform(branches, remoteBranches -> { + List infos = new ArrayList<>(); + + String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranch)) { + infos.add(new BranchInfo(defaultBranch, true)); + } + + remoteBranches.forEach(branch -> { + if (!branch.equals(defaultBranch)) { + infos.add(new BranchInfo(branch, false)); + } + }); + return infos; + }, MoreExecutors.directExecutor())); + } catch (Exception e) { + throw handleException(e); + } + } + + @Data + public static class BranchInfo { + private final String name; + private final boolean isDefault; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 190d68e79e..a588004b79 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -233,7 +233,6 @@ public class RuleChainController extends BaseController { public RuleChain saveRuleChain( @ApiParam(value = "A JSON value representing the rule chain.") @RequestBody RuleChain ruleChain) throws ThingsboardException { - ruleChain.setTenantId(getCurrentUser().getTenantId()); checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN); return tbRuleChainService.save(ruleChain, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index b8c0656bd2..2a64db7385 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,13 +22,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -41,7 +42,6 @@ import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cluster.TbClusterService; import java.util.List; import java.util.Map; @@ -212,7 +212,7 @@ public class EntityActionService { } } - public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, + public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, ActionType actionType, Exception e, Object... additionalInfo) { if (customerId == null || customerId.isNullUid()) { customerId = user.getCustomerId(); @@ -223,6 +223,9 @@ public class EntityActionService { auditLogService.logEntityAction(user.getTenantId(), customerId, user.getId(), user.getName(), entityId, entity, actionType, e, additionalInfo); } + public void sendEntityNotificationMsgToEdgeService(TenantId tenantId, EntityId entityId, EdgeEventActionType action) { + tbClusterService.sendNotificationMsgToEdgeService(tenantId, null, entityId, null, null, action); + } private T extractParameter(Class clazz, int index, Object... additionalInfo) { T result = null; @@ -267,4 +270,5 @@ public class EntityActionService { entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString()); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java new file mode 100644 index 0000000000..55b23a99cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +public class DefaultRateLimitService implements RateLimitService { + + private final TbTenantProfileCache tenantProfileCache; + + private final Map> rateLimits = new ConcurrentHashMap<>(); + + @Override + public boolean checkEntityExportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit); + } + + @Override + public boolean checkEntityImportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit); + } + + private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function rateLimitConfigExtractor) { + String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration() + .map(rateLimitConfigExtractor).orElse(null); + + Map rateLimits = this.rateLimits.get(rateLimitsKey); + if (StringUtils.isEmpty(rateLimitConfig)) { + if (rateLimits != null) { + rateLimits.remove(tenantId); + if (rateLimits.isEmpty()) { + this.rateLimits.remove(rateLimitsKey); + } + } + return true; + } + + if (rateLimits == null) { + rateLimits = new ConcurrentHashMap<>(); + this.rateLimits.put(rateLimitsKey, rateLimits); + } + TbRateLimits rateLimit = rateLimits.get(tenantId); + if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) { + rateLimit = new TbRateLimits(rateLimitConfig); + rateLimits.put(tenantId, rateLimit); + } + + return rateLimit.tryConsume(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java new file mode 100644 index 0000000000..d3d4244ca1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface RateLimitService { + + boolean checkEntityExportLimit(TenantId tenantId); + + boolean checkEntityImportLimit(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index e9999d2dc1..5eaeec0b73 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -26,9 +26,9 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.asset.TbAssetService; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index 34c30d5225..f0a7b6231e 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,9 +49,9 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.device.TbDeviceService; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index fd55e9b321..404e5af5fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -28,9 +28,9 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java index 77dc121e38..7943ec0595 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java @@ -20,7 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java index 0f4ecf7e30..ee10c7859b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java @@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings))); - AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); + AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings))); @@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates))); - AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); + AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates))); @@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { } } - private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) { + private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) { AdminSettings tenantMailSettings = new AdminSettings(); + tenantMailSettings.setTenantId(tenantId); jsonValue.put("useSystemMailSettings", true); tenantMailSettings.setJsonValue(jsonValue); tenantMailSettings.setKey(key); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index 9d87cabb2c..dbbb6ddb1f 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -378,7 +378,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { } List> futures = new ArrayList<>(); for (EntityView entityView : entityViews) { - ListenableFuture future = relationService.checkRelation(tenantId, edge.getId(), entityView.getId(), + ListenableFuture future = relationService.checkRelationAsync(tenantId, edge.getId(), entityView.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE); futures.add(Futures.transformAsync(future, result -> { if (Boolean.TRUE.equals(result)) { @@ -413,11 +413,11 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { } private ListenableFuture saveEdgeEvent(TenantId tenantId, - EdgeId edgeId, - EdgeEventType type, - EdgeEventActionType action, - EntityId entityId, - JsonNode body) { + EdgeId edgeId, + EdgeEventType type, + EdgeEventActionType action, + EntityId entityId, + JsonNode body) { log.trace("Pushing edge event to edge queue. tenantId [{}], edgeId [{}], type [{}], action[{}], entityId [{}], body [{}]", tenantId, edgeId, type, action, entityId, body); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 395ebdeaac..8439c1780b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -64,6 +64,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.resource.TbResourceService; @@ -125,6 +126,8 @@ public abstract class AbstractTbEntityService { @Autowired protected DashboardService dashboardService; @Autowired + protected EntitiesVersionControlService vcService; + @Autowired protected EntityViewService entityViewService; @Autowired protected TelemetrySubscriptionService tsSubService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index 492464f147..4ee8554c4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -46,6 +46,7 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb TenantId tenantId = asset.getTenantId(); try { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); + vcService.autoCommit(user, savedAsset.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), asset, savedAsset.getCustomerId(), actionType, user); return savedAsset; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java index d35cb06a85..536e1be976 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java @@ -40,10 +40,10 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements public Customer save(Customer customer, SecurityUser user) throws ThingsboardException { ActionType actionType = customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = customer.getTenantId(); - CustomerId customerId = customer.getId(); try { Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, customerId, actionType, user); + vcService.autoCommit(user, savedCustomer.getId()); + notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user); return savedCustomer; } catch (Exception e) { notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.CUSTOMER), customer, null, actionType, user, e); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java index f7110c4da7..a475ff77da 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java @@ -48,6 +48,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement TenantId tenantId = dashboard.getTenantId(); try { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); + vcService.autoCommit(user, savedDashboard.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard, null, actionType, user); return savedDashboard; @@ -219,7 +220,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement } @Override - public Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException { + public Dashboard assignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException { ActionType actionType = ActionType.ASSIGNED_TO_EDGE; TenantId tenantId = user.getTenantId(); EdgeId edgeId = edge.getId(); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java index 84dba247ff..313fd737f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java @@ -26,7 +26,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Set; -public interface TbDashboardService extends SimpleTbEntityService { +public interface TbDashboardService extends SimpleTbEntityService { Dashboard assignDashboardToCustomer(DashboardId dashboardId, Customer customer, SecurityUser user) throws ThingsboardException; @@ -40,7 +40,7 @@ public interface TbDashboardService extends SimpleTbEntityService { Dashboard removeDashboardCustomers(Dashboard dashboard, Set customerIds, SecurityUser user) throws ThingsboardException; - Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException; + Dashboard assignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException; Dashboard unassignDashboardFromEdge(Dashboard dashboard, Edge edge, SecurityUser user) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index d74610a462..5199887940 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -54,6 +54,7 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T ActionType actionType = device.getId() == null ? ActionType.ADDED : ActionType.UPDATED; try { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); + vcService.autoCommit(user, savedDevice.getId()); notificationEntityService.notifyCreateOrUpdateDevice(tenantId, savedDevice.getId(), savedDevice.getCustomerId(), savedDevice, oldDevice, actionType, user); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java index ca73b0cb83..3a0c1f1a23 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java @@ -54,7 +54,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple } } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - + vcService.autoCommit(user, savedDeviceProfile.getId()); tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedDeviceProfile.getId(), actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java index 878c6d32bb..2eee167e75 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java @@ -66,19 +66,33 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen @Override public EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException { ActionType actionType = entityView.getId() == null ? ActionType.ADDED : ActionType.UPDATED; + try { + EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView)); + this.updateEntityViewAttributes(user, savedEntityView, existingEntityView); + notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView, + null, actionType, user); + return savedEntityView; + } catch (Exception e) { + notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e); + throw handleException(e); + } + } + + @Override + public void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException { try { List> futures = new ArrayList<>(); - if (existingEntityView != null) { - if (existingEntityView.getKeys() != null && existingEntityView.getKeys().getAttributes() != null) { - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.CLIENT_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SERVER_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SHARED_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); + + if (oldEntityView != null) { + if (oldEntityView.getKeys() != null && oldEntityView.getKeys().getAttributes() != null) { + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.CLIENT_SCOPE, oldEntityView.getKeys().getAttributes().getCs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SERVER_SCOPE, oldEntityView.getKeys().getAttributes().getSs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SHARED_SCOPE, oldEntityView.getKeys().getAttributes().getSh(), user)); } - List tsKeys = existingEntityView.getKeys() != null && existingEntityView.getKeys().getTimeseries() != null ? - existingEntityView.getKeys().getTimeseries() : Collections.emptyList(); - futures.add(deleteLatestFromEntityView(existingEntityView, tsKeys, user)); + List tsKeys = oldEntityView.getKeys() != null && oldEntityView.getKeys().getTimeseries() != null ? + oldEntityView.getKeys().getTimeseries() : Collections.emptyList(); + futures.add(deleteLatestFromEntityView(oldEntityView, tsKeys, user)); } - EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView)); if (savedEntityView.getKeys() != null) { if (savedEntityView.getKeys().getAttributes() != null) { futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), user)); @@ -94,13 +108,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen throw new RuntimeException("Failed to copy attributes to entity view", e); } } - - notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView, - null, actionType, user); - - return savedEntityView; } catch (Exception e) { - notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e); throw handleException(e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java index dd5db9391b..e7e150531b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java @@ -28,6 +28,8 @@ public interface TbEntityViewService { EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException; + void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException; + void delete (EntityView entity, SecurityUser user) throws ThingsboardException; EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer, diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 2a8fc1bf0a..22baed8366 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -27,8 +27,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.queue.TbQueueService; import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.Collections; +import java.util.concurrent.TimeUnit; @Service @TbCoreComponent @@ -38,6 +40,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T private final InstallScripts installScripts; private final TbQueueService tbQueueService; private final TenantProfileService tenantProfileService; + private final EntitiesVersionControlService versionControlService; @Override public Tenant save(Tenant tenant) throws ThingsboardException { @@ -70,6 +73,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T tenantService.deleteTenant(tenantId); tenantProfileCache.evict(tenantId); notificationEntityService.notifyDeleteTenant(tenant); + versionControlService.deleteVersionControlSettings(tenantId).get(1, TimeUnit.MINUTES); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index cece3377b0..50fcb28387 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -265,6 +265,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override public void createAdminSettings() throws Exception { AdminSettings generalSettings = new AdminSettings(); + generalSettings.setTenantId(TenantId.SYS_TENANT_ID); generalSettings.setKey("general"); ObjectNode node = objectMapper.createObjectNode(); node.put("baseUrl", "http://localhost:8080"); @@ -273,6 +274,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); AdminSettings mailSettings = new AdminSettings(); + mailSettings.setTenantId(TenantId.SYS_TENANT_ID); mailSettings.setKey("mail"); node = objectMapper.createObjectNode(); node.put("mailFrom", "ThingsBoard "); diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java index 054f68f71d..c136f97b52 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java @@ -1,5 +1,5 @@ /** - * Copyright © 2016-2021 The Thingsboard Authors + * Copyright © 2016-2022 The Thingsboard Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2806c35cda..6b475f5a5a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -135,6 +135,15 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToVersionControl(TenantId tenantId, TransportProtos.ToVersionControlServiceMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + //TODO: ashvayka + toCoreMsgs.incrementAndGet(); + } + @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2ca47d689f..d071d591a1 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; @@ -75,6 +75,8 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.GitVersionControlQueueService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -117,6 +119,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; @@ -138,7 +141,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService getRuleChainOutputLabels(TenantId tenantId, RuleChainId ruleChainId) { RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId); @@ -169,6 +166,7 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement ActionType actionType = ruleChain.getId() == null ? ActionType.ADDED : ActionType.UPDATED; try { RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); + vcService.autoCommit(user, savedRuleChain.getId()); if (RuleChainType.CORE.equals(savedRuleChain.getType())) { tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedRuleChain.getId(), @@ -222,6 +220,7 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement public RuleChain saveDefaultByName(TenantId tenantId, DefaultRuleChainCreateRequest request, SecurityUser user) throws ThingsboardException { try { RuleChain savedRuleChain = installScripts.createDefaultRuleChain(tenantId, request.getName()); + vcService.autoCommit(user, savedRuleChain.getId()); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedRuleChain.getId(), savedRuleChain, user, ActionType.ADDED, false, null); @@ -281,6 +280,15 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement updatedRuleChains = Collections.emptyList(); } + if (updatedRuleChains.isEmpty()) { + vcService.autoCommit(user, ruleChainMetaData.getRuleChainId()); + } else { + List uuids = new ArrayList<>(updatedRuleChains.size() + 1); + uuids.add(ruleChainMetaData.getRuleChainId().getId()); + updatedRuleChains.forEach(rc -> uuids.add(rc.getId().getId())); + vcService.autoCommit(user, EntityType.RULE_CHAIN, uuids); + } + RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaDataId)); if (RuleChainType.CORE.equals(ruleChain.getType())) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java index 63d3233928..419a60b3b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java @@ -63,7 +63,7 @@ public class SmsTwoFaProvider extends OtpBasedTwoFaProvider { public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { @Override public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { return deviceSessionsCacheEntry.toByteArray(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java new file mode 100644 index 0000000000..b2abe2a581 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.apiusage.RateLimitService; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesExportImportService implements EntitiesExportImportService { + + private final Map> exportServices = new HashMap<>(); + private final Map> importServices = new HashMap<>(); + + private final EntityActionService entityActionService; + private final RelationService relationService; + private final RateLimitService rateLimitService; + + protected static final List SUPPORTED_ENTITY_TYPES = List.of( + EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, + EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE, + EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE + ); + + + @Override + public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) { + throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + + EntityType entityType = entityId.getEntityType(); + EntityExportService> exportService = getExportService(entityType); + + return exportService.getExportData(ctx, entityId); + } + + @Override + public , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException { + if (!rateLimitService.checkEntityImportLimit(ctx.getTenantId())) { + throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + + EntityType entityType = exportData.getEntityType(); + EntityImportService> importService = getImportService(entityType); + + EntityImportResult importResult = importService.importEntity(ctx, exportData); + ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); + + ctx.addReferenceCallback(importResult.getSaveReferencesCallback()); + ctx.addEventCallback(importResult.getSendEventsCallback()); + return importResult; + } + + @Override + public void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException { + for (ThrowingRunnable saveReferencesCallback : ctx.getReferenceCallbacks()) { + saveReferencesCallback.run(); + } + + relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); + + for (EntityRelation relation : ctx.getRelations()) { + entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + } + } + + + @Override + public Comparator getEntityTypeComparatorForImport() { + return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); + } + + + @SuppressWarnings("unchecked") + private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + EntityExportService exportService = exportServices.get(entityType); + if (exportService == null) { + throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); + } + return (EntityExportService) exportService; + } + + @SuppressWarnings("unchecked") + private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + EntityImportService importService = importServices.get(entityType); + if (importService == null) { + throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); + } + return (EntityImportService) importService; + } + + @Autowired + private void setExportServices(DefaultEntityExportService defaultExportService, + Collection> exportServices) { + exportServices.stream() + .sorted(Comparator.comparing(exportService -> exportService.getSupportedEntityTypes().size(), Comparator.reverseOrder())) + .forEach(exportService -> { + exportService.getSupportedEntityTypes().forEach(entityType -> { + this.exportServices.put(entityType, exportService); + }); + }); + SUPPORTED_ENTITY_TYPES.forEach(entityType -> { + this.exportServices.putIfAbsent(entityType, defaultExportService); + }); + } + + @Autowired + private void setImportServices(Collection> importServices) { + importServices.forEach(entityImportService -> { + this.importServices.put(entityImportService.getEntityType(), entityImportService); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java new file mode 100644 index 0000000000..8657528dac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Comparator; + +public interface EntitiesExportImportService { + + , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException; + + , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; + + + void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; + + Comparator getEntityTypeComparatorForImport(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java new file mode 100644 index 0000000000..6e06144ae2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.AccessControlService; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultExportableEntitiesService implements ExportableEntitiesService { + + private final Map> daos = new HashMap<>(); + private final Map> removers = new HashMap<>(); + + private final AccessControlService accessControlService; + + + @Override + public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { + EntityType entityType = externalId.getEntityType(); + Dao dao = getDao(entityType); + + E entity = null; + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + entity = exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + + return entity; + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { + E entity = findEntityById(id); + + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + return entity; + } + + @Override + public , I extends EntityId> E findEntityById(I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + + return dao.findById(TenantId.SYS_TENANT_ID, id.getId()); + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { + Dao dao = getDao(entityType); + + E entity = null; + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + try { + entity = exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); + } catch (UnsupportedOperationException ignored) { + } + } + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + + return entity; + } + + @Override + public , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { + ExportableEntityDao dao = getExportableEntityDao(entityType); + if (dao != null) { + return dao.findByTenantId(tenantId.getId(), pageLink); + } else { + return new PageData<>(); + } + } + + @Override + public I getExternalIdByInternal(I internalId) { + ExportableEntityDao dao = getExportableEntityDao(internalId.getEntityType()); + if (dao != null) { + return dao.getExternalIdByInternal(internalId); + } else { + return null; + } + } + + private boolean belongsToTenant(HasId entity, TenantId tenantId) { + return tenantId.equals(((HasTenantId) entity).getTenantId()); + } + + + @Override + public void removeById(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + BiConsumer entityRemover = removers.get(entityType); + if (entityRemover == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + entityRemover.accept(tenantId, id); + } + + private > ExportableEntityDao getExportableEntityDao(EntityType entityType) { + Dao dao = getDao(entityType); + if (dao instanceof ExportableEntityDao) { + return (ExportableEntityDao) dao; + } else { + return null; + } + } + + @SuppressWarnings("unchecked") + private Dao getDao(EntityType entityType) { + return (Dao) daos.get(entityType); + } + + @Autowired + private void setDaos(Collection> daos) { + daos.forEach(dao -> { + if (dao.getEntityType() != null) { + this.daos.put(dao.getEntityType(), dao); + } + }); + } + + @Autowired + private void setRemovers(CustomerService customerService, AssetService assetService, RuleChainService ruleChainService, + DashboardService dashboardService, DeviceProfileService deviceProfileService, + DeviceService deviceService, WidgetsBundleService widgetsBundleService) { + removers.put(EntityType.CUSTOMER, (tenantId, entityId) -> { + customerService.deleteCustomer(tenantId, (CustomerId) entityId); + }); + removers.put(EntityType.ASSET, (tenantId, entityId) -> { + assetService.deleteAsset(tenantId, (AssetId) entityId); + }); + removers.put(EntityType.RULE_CHAIN, (tenantId, entityId) -> { + ruleChainService.deleteRuleChainById(tenantId, (RuleChainId) entityId); + }); + removers.put(EntityType.DASHBOARD, (tenantId, entityId) -> { + dashboardService.deleteDashboard(tenantId, (DashboardId) entityId); + }); + removers.put(EntityType.DEVICE_PROFILE, (tenantId, entityId) -> { + deviceProfileService.deleteDeviceProfile(tenantId, (DeviceProfileId) entityId); + }); + removers.put(EntityType.DEVICE, (tenantId, entityId) -> { + deviceService.deleteDevice(tenantId, (DeviceId) entityId); + }); + removers.put(EntityType.WIDGETS_BUNDLE, (tenantId, entityId) -> { + widgetsBundleService.deleteWidgetsBundle(tenantId, (WidgetsBundleId) entityId); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java new file mode 100644 index 0000000000..505d1b2a3f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting; + +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +public interface EntityExportService, D extends EntityExportData> { + + D getExportData(EntitiesExportCtx ctx, I entityId) throws ThingsboardException; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java new file mode 100644 index 0000000000..0a99979e8c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; + +public interface ExportableEntitiesService { + + , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId); + + , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id); + + , I extends EntityId> E findEntityById(I id); + + , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); + + , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); + + I getExternalIdByInternal(I internalId); + + void removeById(TenantId tenantId, I id); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java new file mode 100644 index 0000000000..8875491d42 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class AssetExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, Asset asset, EntityExportData exportData) { + asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.ASSET); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java new file mode 100644 index 0000000000..169b7f273c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { + + @Override + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData) throws ThingsboardException { + setRelatedEntities(ctx, entity, (D) exportData); + super.setAdditionalExportData(ctx, entity, exportData); + } + + protected void setRelatedEntities(EntitiesExportCtx ctx, E mainEntity, D exportData) { + } + + protected D newExportData() { + return (D) new EntityExportData(); + } + + ; + + public abstract Set getSupportedEntityTypes(); + + protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet) { + JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java new file mode 100644 index 0000000000..9a3b195b57 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.common.util.RegexUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; +import java.util.UUID; + +@Service +@TbCoreComponent +public class DashboardExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, Dashboard dashboard, EntityExportData exportData) { + if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) { + dashboard.getAssignedCustomers().forEach(customerInfo -> { + customerInfo.setCustomerId(getExternalIdOrElseInternal(ctx, customerInfo.getCustomerId())); + }); + } + for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) { + replaceUuidsRecursively(ctx, entityAlias, Collections.emptySet()); + } + for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { + replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id")); + } + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DASHBOARD); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java new file mode 100644 index 0000000000..7e236370df --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -0,0 +1,184 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@Primary +public class DefaultEntityExportService, D extends EntityExportData> implements EntityExportService { + + @Autowired + @Lazy + protected ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + @Autowired + private AttributesService attributesService; + + @Override + public final D getExportData(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { + D exportData = newExportData(); + + E entity = exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entityId); + if (entity == null) { + throw new IllegalArgumentException(entityId.getEntityType() + " [" + entityId.getId() + "] not found"); + } + + exportData.setEntity(entity); + exportData.setEntityType(entityId.getEntityType()); + setAdditionalExportData(ctx, entity, exportData); + + var externalId = entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + ctx.putExternalId(entityId, externalId); + entity.setId(externalId); + entity.setTenantId(null); + + return exportData; + } + + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData) throws ThingsboardException { + var exportSettings = ctx.getSettings(); + if (exportSettings.isExportRelations()) { + List relations = exportRelations(ctx, entity); + relations.forEach(relation -> { + relation.setFrom(getExternalIdOrElseInternal(ctx, relation.getFrom())); + relation.setTo(getExternalIdOrElseInternal(ctx, relation.getTo())); + }); + exportData.setRelations(relations); + } + if (exportSettings.isExportAttributes()) { + Map> attributes = exportAttributes(ctx, entity); + exportData.setAttributes(attributes); + } + } + + private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { + List relations = new ArrayList<>(); + + List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + relations.addAll(inboundRelations); + + List outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + relations.addAll(outboundRelations); + return relations; + } + + private Map> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { + List scopes; + if (entity.getId().getEntityType() == EntityType.DEVICE) { + scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + } else { + scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + } + Map> attributes = new LinkedHashMap<>(); + scopes.forEach(scope -> { + try { + attributes.put(scope, attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream() + .map(attribute -> { + AttributeExportData attributeExportData = new AttributeExportData(); + attributeExportData.setKey(attribute.getKey()); + attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs()); + attributeExportData.setStrValue(attribute.getStrValue().orElse(null)); + attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null)); + attributeExportData.setLongValue(attribute.getLongValue().orElse(null)); + attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null)); + attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null)); + return attributeExportData; + }) + .collect(Collectors.toList())); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + return attributes; + } + + protected ID getExternalIdOrElseInternal(EntitiesExportCtx ctx, ID internalId) { + if (internalId == null || internalId.isNullUid()) return internalId; + var result = ctx.getExternalId(internalId); + if (result == null) { + result = Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) + .orElse(internalId); + ctx.putExternalId(internalId, result); + } + return result; + } + + protected UUID getExternalIdOrElseInternalByUuid(EntitiesExportCtx ctx, UUID internalUuid) { + for (EntityType entityType : EntityType.values()) { + EntityId internalId; + try { + internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid); + } catch (Exception e) { + continue; + } + EntityId externalId = ctx.getExternalId(internalId); + if (externalId != null) { + return externalId.getId(); + } + } + for (EntityType entityType : EntityType.values()) { + EntityId internalId; + try { + internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid); + } catch (Exception e) { + continue; + } + EntityId externalId = exportableEntitiesService.getExternalIdByInternal(internalId); + if (externalId != null) { + ctx.putExternalId(internalId, externalId); + return externalId.getId(); + } + } + return internalUuid; + } + + protected D newExportData() { + return (D) new EntityExportData(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java new file mode 100644 index 0000000000..bc5136bee6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceExportService extends BaseEntityExportService { + + private final DeviceCredentialsService deviceCredentialsService; + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, Device device, DeviceExportData exportData) { + device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId())); + device.setDeviceProfileId(getExternalIdOrElseInternal(ctx, device.getDeviceProfileId())); + if (ctx.getSettings().isExportCredentials()) { + var credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId()); + credentials.setId(null); + credentials.setDeviceId(null); + exportData.setCredentials(credentials); + } + } + + @Override + protected DeviceExportData newExportData() { + return new DeviceExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java new file mode 100644 index 0000000000..46423c66ef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class DeviceProfileExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData) { + deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId())); + deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE_PROFILE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java new file mode 100644 index 0000000000..3fff890d3a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class EntityViewExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, EntityView entityView, EntityExportData exportData) { + entityView.setEntityId(getExternalIdOrElseInternal(ctx, entityView.getEntityId())); + entityView.setCustomerId(getExternalIdOrElseInternal(ctx, entityView.getCustomerId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.ENTITY_VIEW); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java new file mode 100644 index 0000000000..802a03ab94 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.common.util.RegexUtils; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainExportService extends BaseEntityExportService { + + private final RuleChainService ruleChainService; + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData) { + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), ruleChain.getId()); + Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()) + .forEach(ruleNode -> { + ruleNode.setRuleChainId(null); + ctx.putExternalId(ruleNode.getId(), ruleNode.getExternalId()); + ruleNode.setId(ctx.getExternalId(ruleNode.getId())); + ruleNode.setCreatedTime(0); + ruleNode.setExternalId(null); + replaceUuidsRecursively(ctx, ruleNode.getConfiguration(), Collections.emptySet()); + }); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(getExternalIdOrElseInternal(ctx, ruleChainConnectionInfo.getTargetRuleChainId())); + }); + exportData.setMetaData(metaData); + if (ruleChain.getFirstRuleNodeId() != null) { + ruleChain.setFirstRuleNodeId(ctx.getExternalId(ruleChain.getFirstRuleNodeId())); + } + } + + @Override + protected RuleChainExportData newExportData() { + return new RuleChainExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.RULE_CHAIN); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java new file mode 100644 index 0000000000..d3dd23910f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.List; +import java.util.Set; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class WidgetsBundleExportService extends BaseEntityExportService { + + private final WidgetTypeService widgetTypeService; + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData) { + if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) { + throw new IllegalArgumentException("Export of system Widget Bundles is not allowed"); + } + + List widgets = widgetTypeService.findWidgetTypesDetailsByTenantIdAndBundleAlias(ctx.getTenantId(), widgetsBundle.getAlias()); + exportData.setWidgets(widgets); + } + + @Override + protected WidgetsBundleExportData newExportData() { + return new WidgetsBundleExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.WIDGETS_BUNDLE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java new file mode 100644 index 0000000000..9437f21e60 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +public interface EntityImportService, D extends EntityExportData> { + + EntityImportResult importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException; + + EntityType getEntityType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 81a0068a4d..b92e100401 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,6 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.importing.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -67,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -165,7 +163,7 @@ public abstract class AbstractBulkImportService data) { + private void saveKvs(SecurityUser user, E entity, Map data) { Arrays.stream(BulkImportColumnType.values()) .filter(BulkImportColumnType::isKv) .map(kvType -> { @@ -249,7 +247,7 @@ public abstract class AbstractBulkImportService columnsMappings = request.getMapping().getColumns(); + List columnsMappings = request.getMapping().getColumns(); return records.stream() .map(record -> { EntityData entityData = new EntityData(); @@ -280,7 +278,7 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); - private final Map kvs = new LinkedHashMap<>(); + private final Map kvs = new LinkedHashMap<>(); private int lineNumber; } diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java index 96075eb4c8..24b566e631 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java index 9f8195ca45..e8eac6a9ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java index 651aedeb0b..0626c8e690 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java index 45c2551be2..d48e9a3d23 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java new file mode 100644 index 0000000000..cd9dc17d60 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetImportService extends BaseEntityImportService> { + + private final AssetService assetService; + + @Override + protected void setOwner(TenantId tenantId, Asset asset, IdProvider idProvider) { + asset.setTenantId(tenantId); + asset.setCustomerId(idProvider.getInternalId(asset.getCustomerId())); + } + + @Override + protected Asset prepare(EntitiesImportCtx ctx, Asset asset, Asset old, EntityExportData exportData, IdProvider idProvider) { + return asset; + } + + @Override + protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider) { + return assetService.saveAsset(asset); + } + + @Override + protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) throws ThingsboardException { + super.onEntitySaved(user, savedAsset, oldAsset); + if (oldAsset != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + protected Asset deepCopy(Asset asset) { + return new Asset(asset); + } + + @Override + protected void cleanupForComparison(Asset e) { + super.cleanupForComparison(e); + if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) { + e.setCustomerId(null); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java new file mode 100644 index 0000000000..26c5a869f9 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -0,0 +1,398 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.api.client.util.Objects; +import com.google.common.util.concurrent.FutureCallback; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasCustomerId; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { + + @Autowired + @Lazy + private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + @Autowired + private TelemetrySubscriptionService tsSubService; + @Autowired + protected EntityActionService entityActionService; + @Autowired + protected TbClusterService clusterService; + @Autowired + protected TbNotificationEntityService entityNotificationService; + + @Transactional(rollbackFor = Exception.class) + @Override + public EntityImportResult importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { + EntityImportResult importResult = new EntityImportResult<>(); + ctx.setCurrentImportResult(importResult); + importResult.setEntityType(getEntityType()); + IdProvider idProvider = new IdProvider(ctx, importResult); + + E entity = exportData.getEntity(); + entity.setExternalId(entity.getId()); + + E existingEntity = findExistingEntity(ctx, entity, idProvider); + importResult.setOldEntity(existingEntity); + + setOwner(ctx.getTenantId(), entity, idProvider); + if (existingEntity == null) { + entity.setId(null); + } else { + entity.setId(existingEntity.getId()); + entity.setCreatedTime(existingEntity.getCreatedTime()); + } + + E prepared = prepare(ctx, entity, existingEntity, exportData, idProvider); + + boolean saveOrUpdate = existingEntity == null || compare(ctx, exportData, prepared, existingEntity); + + if (saveOrUpdate) { + E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider); + boolean created = existingEntity == null; + importResult.setCreated(created); + importResult.setUpdated(!created); + importResult.setSavedEntity(savedEntity); + ctx.putInternalId(exportData.getExternalId(), savedEntity.getId()); + } else { + importResult.setSavedEntity(existingEntity); + ctx.putInternalId(exportData.getExternalId(), existingEntity.getId()); + importResult.setUpdatedRelatedEntities(updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider)); + } + + processAfterSaved(ctx, importResult, exportData, idProvider); + + return importResult; + } + + protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { + return false; + } + + @Override + public abstract EntityType getEntityType(); + + protected abstract void setOwner(TenantId tenantId, E entity, IdProvider idProvider); + + protected abstract E prepare(EntitiesImportCtx ctx, E entity, E oldEntity, D exportData, IdProvider idProvider); + + protected boolean compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) { + var newCopy = deepCopy(prepared); + var existingCopy = deepCopy(existing); + cleanupForComparison(newCopy); + cleanupForComparison(existingCopy); + var result = !newCopy.equals(existingCopy); + if (result) { + log.debug("[{}] Found update.", prepared.getId()); + log.debug("[{}] From: {}", prepared.getId(), newCopy); + log.debug("[{}] To: {}", prepared.getId(), existingCopy); + } + return result; + } + + protected abstract E deepCopy(E e); + + protected void cleanupForComparison(E e) { + e.setTenantId(null); + e.setCreatedTime(0); + } + + protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider); + + + protected void processAfterSaved(EntitiesImportCtx ctx, EntityImportResult importResult, D exportData, IdProvider idProvider) throws ThingsboardException { + E savedEntity = importResult.getSavedEntity(); + E oldEntity = importResult.getOldEntity(); + + if (importResult.isCreated() || importResult.isUpdated()) { + importResult.addSendEventsCallback(() -> onEntitySaved(ctx.getUser(), savedEntity, oldEntity)); + } + + if (ctx.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(ctx, exportData.getRelations(), importResult, idProvider); + } + if (ctx.isSaveAttributes() && exportData.getAttributes() != null) { + if (exportData.getAttributes().values().stream().anyMatch(d -> !d.isEmpty())) { + importResult.setUpdatedRelatedEntities(true); + } + importAttributes(ctx.getUser(), exportData.getAttributes(), importResult); + } + } + + private void importRelations(EntitiesImportCtx ctx, List relations, EntityImportResult importResult, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + for (EntityRelation relation : relations) { + if (!relation.getTo().equals(entity.getId())) { + relation.setTo(idProvider.getInternalId(relation.getTo())); + } + if (!relation.getFrom().equals(entity.getId())) { + relation.setFrom(idProvider.getInternalId(relation.getFrom())); + } + } + + Map relationsMap = new LinkedHashMap<>(); + relations.forEach(r -> relationsMap.put(r, r)); + + if (importResult.getOldEntity() != null) { + List existingRelations = new ArrayList<>(); + existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + + for (EntityRelation existingRelation : existingRelations) { + EntityRelation relation = relationsMap.get(existingRelation); + if (relation == null) { + importResult.setUpdatedRelatedEntities(true); + relationService.deleteRelation(tenantId, existingRelation); + importResult.addSendEventsCallback(() -> { + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + }); + } else if (Objects.equal(relation.getAdditionalInfo(), existingRelation.getAdditionalInfo())) { + relationsMap.remove(relation); + } + } + } + if (!relationsMap.isEmpty()) { + importResult.setUpdatedRelatedEntities(true); + ctx.addRelations(relationsMap.values()); + } + }); + } + + private void importAttributes(SecurityUser user, Map> attributes, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + attributes.forEach((scope, attributesExportData) -> { + List attributeKvEntries = attributesExportData.stream() + .map(attributeExportData -> { + KvEntry kvEntry; + String key = attributeExportData.getKey(); + if (attributeExportData.getStrValue() != null) { + kvEntry = new StringDataEntry(key, attributeExportData.getStrValue()); + } else if (attributeExportData.getBooleanValue() != null) { + kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue()); + } else if (attributeExportData.getDoubleValue() != null) { + kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue()); + } else if (attributeExportData.getLongValue() != null) { + kvEntry = new LongDataEntry(key, attributeExportData.getLongValue()); + } else if (attributeExportData.getJsonValue() != null) { + kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue()); + } else { + throw new IllegalArgumentException("Invalid attribute export data"); + } + return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs()); + }) + .collect(Collectors.toList()); + // fixme: attributes are saved outside the transaction + tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void unused) { + } + + @Override + public void onFailure(Throwable thr) { + log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr); + } + }); + }); + }); + } + + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, + savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); + } + + + @SuppressWarnings("unchecked") + protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { + return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId()))) + .or(() -> { + if (ctx.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName())); + } else { + return Optional.empty(); + } + }) + .orElse(null); + } + + @SuppressWarnings("unchecked") + private HasId findInternalEntity(TenantId tenantId, ID externalId) { + return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) + .orElseThrow(() -> new MissingEntityException(externalId)); + } + + + @SuppressWarnings("unchecked") + @RequiredArgsConstructor + protected class IdProvider { + private final EntitiesImportCtx ctx; + private final EntityImportResult importResult; + + public ID getInternalId(ID externalId) { + return getInternalId(externalId, true); + } + + public ID getInternalId(ID externalId, boolean throwExceptionIfNotFound) { + if (externalId == null || externalId.isNullUid()) return null; + + EntityId localId = ctx.getInternalId(externalId); + if (localId != null) { + return (ID) localId; + } + + HasId entity; + try { + entity = findInternalEntity(ctx.getTenantId(), externalId); + } catch (Exception e) { + if (throwExceptionIfNotFound) { + throw e; + } else { + importResult.setUpdatedAllExternalIds(false); + return null; + } + } + ctx.putInternalId(externalId, entity.getId()); + return entity.getId(); + } + + public Optional getInternalIdByUuid(UUID externalUuid, boolean fetchAllUUIDs, Set hints) { + if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty(); + + for (EntityType entityType : EntityType.values()) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (!externalIdOpt.isPresent()) { + continue; + } + EntityId internalId = ctx.getInternalId(externalIdOpt.get()); + if (internalId != null) { + return Optional.of(internalId); + } + } + + if (fetchAllUUIDs) { + for (EntityType entityType : hints) { + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; + } + for (EntityType entityType : EntityType.values()) { + if (hints.contains(entityType)) { + continue; + } + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; + } + } + + importResult.setUpdatedAllExternalIds(false); + return Optional.empty(); + } + + private Optional lookupInDb(UUID externalUuid, EntityType entityType) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (externalIdOpt.isEmpty() || ctx.isNotFound(externalIdOpt.get())) { + return Optional.empty(); + } + EntityId internalId = getInternalId(externalIdOpt.get(), false); + if (internalId != null) { + return Optional.of(internalId); + } else { + ctx.registerNotFound(externalIdOpt.get()); + } + return Optional.empty(); + } + + private Optional buildEntityId(EntityType entityType, UUID externalUuid) { + try { + return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, externalUuid)); + } catch (Exception e) { + return Optional.empty(); + } + } + + } + + protected T getOldEntityField(O oldEntity, Function getter) { + return oldEntity == null ? null : getter.apply(oldEntity); + } + + protected void replaceIdsRecursively(EntitiesImportCtx ctx, IdProvider idProvider, JsonNode entityAlias, Set skipFieldsSet, LinkedHashSet hints) { + JacksonUtil.replaceUuidsRecursively(entityAlias, skipFieldsSet, + uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints).map(EntityId::getId).orElse(uuid)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java new file mode 100644 index 0000000000..1dc476340f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.customer.CustomerDao; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CustomerImportService extends BaseEntityImportService> { + + private final CustomerService customerService; + private final CustomerDao customerDao; + + @Override + protected void setOwner(TenantId tenantId, Customer customer, IdProvider idProvider) { + customer.setTenantId(tenantId); + } + + @Override + protected Customer prepare(EntitiesImportCtx ctx, Customer customer, Customer old, EntityExportData exportData, IdProvider idProvider) { + if (customer.isPublic()) { + Customer publicCustomer = customerService.findOrCreatePublicCustomer(ctx.getTenantId()); + publicCustomer.setExternalId(customer.getExternalId()); + return publicCustomer; + } else { + return customer; + } + } + + @Override + protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData exportData, IdProvider idProvider) { + if (!customer.isPublic()) { + return customerService.saveCustomer(customer); + } else { + return customerDao.save(ctx.getTenantId(), customer); + } + } + + @Override + protected Customer deepCopy(Customer customer) { + return new Customer(customer); + } + + @Override + protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException { + super.onEntitySaved(user, savedCustomer, oldCustomer); + if (oldCustomer != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java new file mode 100644 index 0000000000..b6de84bfff --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -0,0 +1,139 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.common.util.RegexUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DashboardImportService extends BaseEntityImportService> { + + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.DASHBOARD, EntityType.DEVICE, EntityType.ASSET)); + + private final DashboardService dashboardService; + + + @Override + protected void setOwner(TenantId tenantId, Dashboard dashboard, IdProvider idProvider) { + dashboard.setTenantId(tenantId); + } + + @Override + protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard, IdProvider idProvider) { + Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard, idProvider); + if (existingDashboard == null && ctx.isFindExistingByName()) { + existingDashboard = dashboardService.findTenantDashboardsByTitle(ctx.getTenantId(), dashboard.getName()).stream().findFirst().orElse(null); + } + return existingDashboard; + } + + @Override + protected Dashboard prepare(EntitiesImportCtx ctx, Dashboard dashboard, Dashboard old, EntityExportData exportData, IdProvider idProvider) { + for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) { + replaceIdsRecursively(ctx, idProvider, entityAlias, Collections.emptySet(), HINTS); + } + for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { + replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id"), HINTS); + } + return dashboard; + } + + @Override + protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); + + Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() + .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) + .collect(Collectors.toSet()); + + if (dashboard.getId() == null) { + dashboard.setAssignedCustomers(assignedCustomers); + dashboard = dashboardService.saveDashboard(dashboard); + for (ShortCustomerInfo customerInfo : assignedCustomers) { + dashboard = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerInfo.getCustomerId()); + } + } else { + Set existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) + .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set newAssignedCustomers = assignedCustomers.stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + + Set toUnassign = new HashSet<>(existingAssignedCustomers); + toUnassign.removeAll(newAssignedCustomers); + for (CustomerId customerId : toUnassign) { + assignedCustomers = dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); + } + + Set toAssign = new HashSet<>(newAssignedCustomers); + toAssign.removeAll(existingAssignedCustomers); + for (CustomerId customerId : toAssign) { + assignedCustomers = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); + } + dashboard.setAssignedCustomers(assignedCustomers); + dashboard = dashboardService.saveDashboard(dashboard); + } + return dashboard; + } + + @Override + protected Dashboard deepCopy(Dashboard dashboard) { + return new Dashboard(dashboard); + } + + @Override + protected void onEntitySaved(SecurityUser user, Dashboard savedDashboard, Dashboard oldDashboard) throws ThingsboardException { + super.onEntitySaved(user, savedDashboard, oldDashboard); + if (oldDashboard != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java new file mode 100644 index 0000000000..c699f466d7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceImportService extends BaseEntityImportService { + + private final DeviceService deviceService; + private final DeviceCredentialsService credentialsService; + + @Override + protected void setOwner(TenantId tenantId, Device device, IdProvider idProvider) { + device.setTenantId(tenantId); + device.setCustomerId(idProvider.getInternalId(device.getCustomerId())); + } + + @Override + protected Device prepare(EntitiesImportCtx ctx, Device device, Device old, DeviceExportData exportData, IdProvider idProvider) { + device.setDeviceProfileId(idProvider.getInternalId(device.getDeviceProfileId())); + device.setFirmwareId(getOldEntityField(old, Device::getFirmwareId)); + device.setSoftwareId(getOldEntityField(old, Device::getSoftwareId)); + return device; + } + + @Override + protected Device deepCopy(Device d) { + return new Device(d); + } + + @Override + protected void cleanupForComparison(Device e) { + super.cleanupForComparison(e); + if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) { + e.setCustomerId(null); + } + } + + @Override + protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) { + if (exportData.getCredentials() != null && ctx.isSaveCredentials()) { + exportData.getCredentials().setId(null); + exportData.getCredentials().setDeviceId(null); + return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); + } else { + return deviceService.saveDevice(device); + } + } + + @Override + protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, Device prepared, DeviceExportData exportData, IdProvider idProvider) { + boolean updated = super.updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider); + var credentials = exportData.getCredentials(); + if (credentials != null && ctx.isSaveCredentials()) { + var existing = credentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), prepared.getId()); + credentials.setId(existing.getId()); + credentials.setDeviceId(prepared.getId()); + if (!existing.equals(credentials)) { + credentialsService.updateDeviceCredentials(ctx.getTenantId(), credentials); + updated = true; + } + } + return updated; + } + + @Override + protected void onEntitySaved(SecurityUser user, Device savedDevice, Device oldDevice) throws ThingsboardException { + super.onEntitySaved(user, savedDevice, oldDevice); + clusterService.onDeviceUpdated(savedDevice, oldDevice); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java new file mode 100644 index 0000000000..0944792751 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ota.OtaPackageStateService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Objects; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceProfileImportService extends BaseEntityImportService> { + + private final DeviceProfileService deviceProfileService; + private final OtaPackageStateService otaPackageStateService; + + @Override + protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) { + deviceProfile.setTenantId(tenantId); + } + + @Override + protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData exportData, IdProvider idProvider) { + deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); + deviceProfile.setFirmwareId(getOldEntityField(old, DeviceProfile::getFirmwareId)); + deviceProfile.setSoftwareId(getOldEntityField(old, DeviceProfile::getSoftwareId)); + return deviceProfile; + } + + @Override + protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + @Override + protected void onEntitySaved(SecurityUser user, DeviceProfile savedDeviceProfile, DeviceProfile oldDeviceProfile) throws ThingsboardException { + super.onEntitySaved(user, savedDeviceProfile, oldDeviceProfile); + clusterService.onDeviceProfileChange(savedDeviceProfile, null); + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + otaPackageStateService.update(savedDeviceProfile, + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); + } + + @Override + protected DeviceProfile deepCopy(DeviceProfile deviceProfile) { + return new DeviceProfile(deviceProfile); + } + + @Override + protected void cleanupForComparison(DeviceProfile deviceProfile) { + super.cleanupForComparison(deviceProfile); + deviceProfile.setFirmwareId(null); + deviceProfile.setSoftwareId(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java new file mode 100644 index 0000000000..57d22cf7be --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class EntityViewImportService extends BaseEntityImportService> { + + private final EntityViewService entityViewService; + + @Lazy + @Autowired + private TbEntityViewService tbEntityViewService; + + @Override + protected void setOwner(TenantId tenantId, EntityView entityView, IdProvider idProvider) { + entityView.setTenantId(tenantId); + entityView.setCustomerId(idProvider.getInternalId(entityView.getCustomerId())); + } + + @Override + protected EntityView prepare(EntitiesImportCtx ctx, EntityView entityView, EntityView old, EntityExportData exportData, IdProvider idProvider) { + entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId())); + return entityView; + } + + @Override + protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData exportData, IdProvider idProvider) { + return entityViewService.saveEntityView(entityView); + } + + @Override + protected void onEntitySaved(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException { + tbEntityViewService.updateEntityViewAttributes(user, savedEntityView, oldEntityView); + super.onEntitySaved(user, savedEntityView, oldEntityView); + if (oldEntityView != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedEntityView.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + protected EntityView deepCopy(EntityView entityView) { + return new EntityView(entityView); + } + + @Override + protected void cleanupForComparison(EntityView e) { + super.cleanupForComparison(e); + if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) { + e.setCustomerId(null); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java new file mode 100644 index 0000000000..1e869429d8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +public class ImportServiceException extends RuntimeException{ + private static final long serialVersionUID = -4932715239522125041L; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java new file mode 100644 index 0000000000..a0a961bfef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; + +public class MissingEntityException extends ImportServiceException { + + private static final long serialVersionUID = 3669135386955906022L; + @Getter + private final EntityId entityId; + + public MissingEntityException(EntityId entityId) { + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java new file mode 100644 index 0000000000..916c10844f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -0,0 +1,152 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.rule.RuleNodeDao; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.common.util.RegexUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +@Slf4j +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainImportService extends BaseEntityImportService { + + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); + + private final RuleChainService ruleChainService; + private final RuleNodeDao ruleNodeDao; + + @Override + protected void setOwner(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) { + ruleChain.setTenantId(tenantId); + } + + @Override + protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain, IdProvider idProvider) { + RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain, idProvider); + if (existingRuleChain == null && ctx.isFindExistingByName()) { + existingRuleChain = ruleChainService.findTenantRuleChainsByTypeAndName(ctx.getTenantId(), ruleChain.getType(), ruleChain.getName()).stream().findFirst().orElse(null); + } + return existingRuleChain; + } + + @Override + protected RuleChain prepare(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChain old, RuleChainExportData exportData, IdProvider idProvider) { + RuleChainMetaData metaData = exportData.getMetaData(); + List ruleNodes = Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()); + if (old != null) { + List nodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + List existing = ruleNodeDao.findByExternalIds(old.getId(), nodeIds); + existing.forEach(node -> ctx.putInternalId(node.getExternalId(), node.getId())); + ruleNodes.forEach(node -> { + node.setRuleChainId(old.getId()); + node.setExternalId(node.getId()); + node.setId((RuleNodeId) ctx.getInternalId(node.getId())); + }); + } else { + ruleNodes.forEach(node -> { + node.setRuleChainId(null); + node.setExternalId(node.getId()); + node.setId(null); + }); + } + + ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), Collections.emptySet(), HINTS)); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false)); + }); + if (ruleChain.getFirstRuleNodeId() != null) { + ruleChain.setFirstRuleNodeId((RuleNodeId) ctx.getInternalId(ruleChain.getFirstRuleNodeId())); + } + return ruleChain; + } + + @Override + protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider) { + ruleChain = ruleChainService.saveRuleChain(ruleChain); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + exportData.getMetaData().setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(ctx.getTenantId(), exportData.getMetaData()); + return ruleChainService.findRuleChainById(ctx.getTenantId(), ruleChain.getId()); + } else { + return ruleChain; + } + } + + @Override + protected boolean compare(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) { + boolean different = super.compare(ctx, exportData, prepared, existing); + if (!different) { + RuleChainMetaData newMD = exportData.getMetaData(); + RuleChainMetaData existingMD = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), prepared.getId()); + existingMD.setRuleChainId(null); + different = newMD.equals(existingMD); + } + return different; + } + + @Override + protected void onEntitySaved(SecurityUser user, RuleChain savedRuleChain, RuleChain oldRuleChain) throws ThingsboardException { + super.onEntitySaved(user, savedRuleChain, oldRuleChain); + if (savedRuleChain.getType() == RuleChainType.CORE) { + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { + entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + protected RuleChain deepCopy(RuleChain ruleChain) { + return new RuleChain(ruleChain); + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java new file mode 100644 index 0000000000..550f00952a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData; +import org.thingsboard.server.common.data.widget.BaseWidgetType; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class WidgetsBundleImportService extends BaseEntityImportService { + + private final WidgetsBundleService widgetsBundleService; + private final WidgetTypeService widgetTypeService; + + @Override + protected void setOwner(TenantId tenantId, WidgetsBundle widgetsBundle, IdProvider idProvider) { + widgetsBundle.setTenantId(tenantId); + } + + @Override + protected WidgetsBundle prepare(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundle old, WidgetsBundleExportData exportData, IdProvider idProvider) { + return widgetsBundle; + } + + @Override + protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider) { + WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); + if (widgetsBundle.getId() == null) { + for (WidgetTypeDetails widget : exportData.getWidgets()) { + widget.setId(null); + widget.setTenantId(ctx.getTenantId()); + widget.setBundleAlias(savedWidgetsBundle.getAlias()); + widgetTypeService.saveWidgetType(widget); + } + } else { + Map existingWidgets = widgetTypeService.findWidgetTypesInfosByTenantIdAndBundleAlias(ctx.getTenantId(), savedWidgetsBundle.getAlias()).stream() + .collect(Collectors.toMap(BaseWidgetType::getAlias, w -> w)); + for (WidgetTypeDetails widget : exportData.getWidgets()) { + WidgetTypeInfo existingWidget; + if ((existingWidget = existingWidgets.remove(widget.getAlias())) != null) { + widget.setId(existingWidget.getId()); + widget.setCreatedTime(existingWidget.getCreatedTime()); + } else { + widget.setId(null); + } + widget.setTenantId(ctx.getTenantId()); + widget.setBundleAlias(savedWidgetsBundle.getAlias()); + widgetTypeService.saveWidgetType(widget); + } + existingWidgets.values().stream() + .map(BaseWidgetType::getId) + .forEach(widgetTypeId -> widgetTypeService.deleteWidgetType(ctx.getTenantId(), widgetTypeId)); + } + return savedWidgetsBundle; + } + + @Override + protected void onEntitySaved(SecurityUser user, WidgetsBundle savedWidgetsBundle, WidgetsBundle oldWidgetsBundle) throws ThingsboardException { + super.onEntitySaved(user, savedWidgetsBundle, oldWidgetsBundle); + entityNotificationService.notifySendMsgToEdgeService(user.getTenantId(), savedWidgetsBundle.getId(), + oldWidgetsBundle == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + } + + @Override + protected WidgetsBundle deepCopy(WidgetsBundle widgetsBundle) { + return new WidgetsBundle(widgetsBundle); + } + + @Override + public EntityType getEntityType() { + return EntityType.WIDGETS_BUNDLE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..1f6c174fdc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -0,0 +1,554 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.TbStopWatch; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityLoadError; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; +import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx; +import org.thingsboard.server.service.sync.vc.data.ReimportTask; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.google.common.util.concurrent.Futures.transform; +import static com.google.common.util.concurrent.Futures.transformAsync; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + + private final TbRepositorySettingsService repositorySettingsService; + private final TbAutoCommitSettingsService autoCommitSettingsService; + private final GitVersionControlQueueService gitServiceQueue; + private final EntitiesExportImportService exportImportService; + private final ExportableEntitiesService exportableEntitiesService; + private final TbNotificationEntityService entityNotificationService; + private final TransactionTemplate transactionTemplate; + + private ListeningExecutorService executor; + + @Value("${vc.thread_pool_size:4}") + private int threadPoolSize; + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class)); + } + + @PreDestroy + public void shutdown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + @SuppressWarnings("UnstableApiUsage") + @Override + public ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { + var pendingCommit = gitServiceQueue.prepareCommit(user, request); + + return transformAsync(pendingCommit, commit -> { + List> gitFutures = new ArrayList<>(); + switch (request.getType()) { + case SINGLE_ENTITY: { + handleSingleEntityRequest(new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request)); + break; + } + case COMPLEX: { + handleComplexRequest(new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request)); + break; + } + } + return transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor); + }, executor); + } + + private void handleSingleEntityRequest(SimpleEntitiesExportCtx ctx) throws Exception { + ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId())); + } + + private void handleComplexRequest(ComplexEntitiesExportCtx parentCtx) { + ComplexVersionCreateRequest request = parentCtx.getRequest(); + request.getEntityTypes().forEach((entityType, config) -> { + EntityTypeExportCtx ctx = new EntityTypeExportCtx(parentCtx, config, request.getSyncStrategy(), entityType); + if (ctx.isOverwrite()) { + ctx.add(gitServiceQueue.deleteAll(ctx.getCommit(), entityType)); + } + + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink) + , 100, entity -> { + try { + ctx.add(saveEntityData(ctx, entity.getId())); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + }); + } + + private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId); + return gitServiceQueue.addToCommit(ctx.getCommit(), entityData); + } + + @Override + public ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, externalId, pageLink); + } + + @Override + public ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, entityType, pageLink); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, pageLink); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId, entityType); + } + + @Override + public ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId); + } + + @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) + @Override + public ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + VersionLoadConfig config = versionLoadRequest.getConfig(); + ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + return Futures.transform(future, entityData -> doInTemplate(user, request, ctx -> loadSingleEntity(ctx, config, entityData)), executor); + } + case ENTITY_TYPE: { + EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; + return executor.submit(() -> doInTemplate(user, request, ctx -> loadMultipleEntities(ctx, versionLoadRequest))); + } + default: + throw new IllegalArgumentException("Unsupported version load request"); + } + } + + private VersionLoadResult doInTemplate(SecurityUser user, VersionLoadRequest request, Function function) { + try { + EntitiesImportCtx ctx = new EntitiesImportCtx(user, request.getVersionId()); + VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx)); + try { + for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { + throwingRunnable.run(); + } + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + return result; + } catch (LoadEntityException e) { + return onError(e.getData(), e.getCause()); + } catch (Exception e) { + log.info("[{}] Failed to process request [{}] due to: ", user.getTenantId(), request, e); + throw e; + } + } + + private VersionLoadResult loadSingleEntity(EntitiesImportCtx ctx, VersionLoadConfig config, EntityExportData entityData) { + try { + ctx.setSettings(EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) + .findExistingByName(false) + .build()); + ctx.setFinalImportAttempt(true); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); + + exportImportService.saveReferencesAndRelations(ctx); + + return VersionLoadResult.success(EntityTypeLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + } + + @SneakyThrows + private VersionLoadResult loadMultipleEntities(EntitiesImportCtx ctx, EntityTypeVersionLoadRequest request) { + var sw = TbStopWatch.create("before"); + + List entityTypes = request.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + for (EntityType entityType : entityTypes) { + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); + sw.startNew("Entities " + entityType.name()); + ctx.setSettings(getEntityImportSettings(request, entityType)); + importEntities(ctx, entityType); + } + + sw.startNew("Reimport"); + reimport(ctx); + + sw.startNew("Remove Others"); + request.getEntityTypes().keySet().stream() + .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> removeOtherEntities(ctx, entityType)); + + sw.startNew("References and Relations"); + + exportImportService.saveReferencesAndRelations(ctx); + + sw.stop(); + for (var task : sw.getTaskInfo()) { + log.info("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); + } + log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); + return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())); + } + + private EntityImportSettings getEntityImportSettings(EntityTypeVersionLoadRequest request, EntityType entityType) { + var config = request.getEntityTypes().get(entityType); + return EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void importEntities(EntitiesImportCtx ctx, EntityType entityType) { + int limit = 100; + int offset = 0; + List entityDataList; + do { + try { + entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (EntityExportData entityData : entityDataList) { + EntityExportData reimportBackup = JacksonUtil.clone(entityData); + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); + EntityImportResult importResult; + try { + importResult = exportImportService.importEntity(ctx, entityData); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + if (!importResult.isUpdatedAllExternalIds()) { + ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(reimportBackup, ctx.getSettings())); + continue; + } + + registerResult(ctx, entityType, importResult); + ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) + .add(importResult.getSavedEntity().getId()); + } + offset += limit; + } while (entityDataList.size() == limit); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void reimport(EntitiesImportCtx ctx) { + ctx.setFinalImportAttempt(true); + ctx.getToReimport().forEach((externalId, task) -> { + try { + EntityExportData entityData = task.getData(); + var settings = task.getSettings(); + ctx.setSettings(settings); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); + + registerResult(ctx, externalId.getEntityType(), importResult); + ctx.getImportedEntities().computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>()) + .add(importResult.getSavedEntity().getId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) { + exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId()); + + ctx.addEventCallback(() -> { + entityNotificationService.notifyDeleteEntity(ctx.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, ctx.getUser()); + }); + ctx.registerDeleted(entityType); + } + }); + } + + private VersionLoadResult onError(EntityExportData entityData, Throwable e) { + return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e)); + } + + private Optional analyze(Throwable e, EntityExportData entityData) { + if (e == null) { + return Optional.empty(); + } else { + if (e instanceof DeviceCredentialsValidationException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId()))); + } else if (e instanceof MissingEntityException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId()))); + } else { + return analyze(e.getCause(), entityData); + } + } + } + + @Override + public ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception { + HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); + + EntityId externalId = ((ExportableEntity) entity).getExternalId(); + if (externalId == null) externalId = entityId; + + return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + otherVersion -> { + SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder() + .exportRelations(otherVersion.hasRelations()) + .exportAttributes(otherVersion.hasAttributes()) + .exportCredentials(otherVersion.hasCredentials()) + .build()); + EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId); + return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), + rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) { + return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), + entity -> new EntityDataInfo(entity.hasRelations(), entity.hasAttributes(), entity.hasCredentials()), MoreExecutors.directExecutor()); + } + + + @Override + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + return gitServiceQueue.listBranches(tenantId); + } + + @Override + public RepositorySettings getVersionControlSettings(TenantId tenantId) { + return repositorySettingsService.get(tenantId); + } + + @Override + public ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings) { + var restoredSettings = this.repositorySettingsService.restore(tenantId, versionControlSettings); + try { + var future = gitServiceQueue.initRepository(tenantId, restoredSettings); + return Futures.transform(future, f -> repositorySettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); + } catch (Exception e) { + log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e); + throw new RuntimeException("Failed to init repository!", e); + } + } + + @Override + public ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception { + if (repositorySettingsService.delete(tenantId)) { + return gitServiceQueue.clearRepository(tenantId); + } else { + return Futures.immediateFuture(null); + } + } + + @Override + public ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws ThingsboardException { + settings = this.repositorySettingsService.restore(tenantId, settings); + try { + return gitServiceQueue.testRepository(tenantId, settings); + } catch (Exception e) { + throw new ThingsboardException(String.format("Unable to access repository: %s", getCauseMessage(e)), + ThingsboardErrorCode.GENERAL); + } + } + + @Override + public ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } + var entityType = entityId.getEntityType(); + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; + } + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setEntityId(entityId); + vcr.setConfig(autoCommitConfig); + return saveEntitiesVersion(user, vcr); + } + + @Override + public ListenableFuture autoCommit(SecurityUser user, EntityType entityType, List entityIds) throws Exception { + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; + } + ComplexVersionCreateRequest vcr = new ComplexVersionCreateRequest(); + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setSyncStrategy(SyncStrategy.MERGE); + + EntityTypeVersionCreateConfig vcrConfig = new EntityTypeVersionCreateConfig(); + vcrConfig.setEntityIds(entityIds); + vcr.setEntityTypes(Collections.singletonMap(entityType, vcrConfig)); + return saveEntitiesVersion(user, vcr); + } + + private String getCauseMessage(Exception e) { + String message; + if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) { + message = e.getCause().getMessage(); + } else { + message = e.getMessage(); + } + return message; + } + + private void registerResult(EntitiesImportCtx ctx, EntityType entityType, EntityImportResult importResult) { + if (importResult.isCreated()) { + ctx.registerResult(entityType, true); + } else if (importResult.isUpdated() || importResult.isUpdatedRelatedEntities()) { + ctx.registerResult(entityType, false); + } + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java new file mode 100644 index 0000000000..f5585fac39 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -0,0 +1,526 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.ByteString; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GenericRepositoryRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ContentsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntityContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListBranchesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListEntitiesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListVersionsGitRequest; +import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; +import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final TbClusterService clusterService; + private final DataDecodingEncodingService encodingService; + private final DefaultEntitiesVersionControlService entitiesVersionControlService; + private final SchedulerComponent scheduler; + + private final Map> pendingRequestMap = new HashMap<>(); + + @Value("${queue.vc.request-timeout:60000}") + private int requestTimeout; + + public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, + DataDecodingEncodingService encodingService, + @Lazy DefaultEntitiesVersionControlService entitiesVersionControlService, + SchedulerComponent scheduler) { + this.serviceInfoProvider = serviceInfoProvider; + this.clusterService = clusterService; + this.encodingService = encodingService; + this.entitiesVersionControlService = entitiesVersionControlService; + this.scheduler = scheduler; + } + + @Override + public ListenableFuture prepareCommit(User user, VersionCreateRequest request) { + SettableFuture future = SettableFuture.create(); + + CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() + ).build(), wrap(future, commit)); + return future; + } + + @Override + public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); + String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); + + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJson(entityDataJson).build() + ).build() + ).build(), wrap(future, null)); + return future; + } + + @Override + public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityType, null); + + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setDeleteMsg( + TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() + ).build() + ).build(), wrap(future, null)); + + return future; + } + + @Override + public ListenableFuture push(CommitGitRequest commit) { + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setPushMsg( + TransportProtos.PushMsg.newBuilder().build() + ).build() + ).build(), wrap(commit.getFuture())); + + return commit.getFuture(); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink) { + + return listVersions(tenantId, + applyPageLinkParameters( + ListVersionsRequestMsg.newBuilder() + .setBranchName(branch), + pageLink + ).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, + applyPageLinkParameters( + ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityType.name()), + pageLink + ).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, + applyPageLinkParameters( + ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), + pageLink + ).build()); + } + + private ListVersionsRequestMsg.Builder applyPageLinkParameters(ListVersionsRequestMsg.Builder builder, PageLink pageLink) { + builder.setPageSize(pageLink.getPageSize()) + .setPage(pageLink.getPage()); + if (pageLink.getTextSearch() != null) { + builder.setTextSearch(pageLink.getTextSearch()); + } + if (pageLink.getSortOrder() != null) { + if (pageLink.getSortOrder().getProperty() != null) { + builder.setSortProperty(pageLink.getSortOrder().getProperty()); + } + if (pageLink.getSortOrder().getDirection() != null) { + builder.setSortDirection(pageLink.getSortOrder().getDirection().name()); + } + } + return builder; + } + + private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { + ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListVersionRequest(requestMsg)); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .build()); + } + + private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { + ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListEntitiesRequest(requestMsg)); + } + + @Override + public ListenableFuture> listBranches(TenantId tenantId) { + ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); + } + + @Override + public ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2) { + String path = entityType != null ? getRelativePath(entityType, externalId) : ""; + VersionsDiffGitRequest request = new VersionsDiffGitRequest(tenantId, path, versionId1, versionId2); + return sendRequest(request, builder -> builder.setVersionsDiffRequest(TransportProtos.VersionsDiffRequestMsg.newBuilder() + .setPath(request.getPath()) + .setVersionId1(request.getVersionId1()) + .setVersionId2(request.getVersionId2()) + .build())); + } + + @Override + public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { + ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); + return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() + .setContent1(content1) + .setContent2(content2))); + } + + @Override + @SuppressWarnings("rawtypes") + public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { + EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() + , wrap(request.getFuture())); + return request.getFuture(); + } + + private void registerAndSend(PendingGitRequest request, + Function enrichFunction, TbQueueCallback callback) { + registerAndSend(request, enrichFunction, null, callback); + } + + private void registerAndSend(PendingGitRequest request, + Function enrichFunction, RepositorySettings settings, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + var requestBody = enrichFunction.apply(newRequestProto(request, settings)); + log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); + clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); + request.setTimeoutTask(scheduler.schedule(() -> { + processTimeout(request.getRequestId()); + }, requestTimeout, TimeUnit.MILLISECONDS)); + } else { + throw new RuntimeException("Future is already done!"); + } + } + + private ListenableFuture sendRequest(PendingGitRequest request, Consumer enrichFunction) { + registerAndSend(request, builder -> { + enrichFunction.accept(builder); + return builder.build(); + }, wrap(request.getFuture())); + return request.getFuture(); + } + + @Override + @SuppressWarnings("rawtypes") + public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { + EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); + + registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityType.name()) + .setOffset(offset) + .setLimit(limit) + ).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , settings, wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder + .setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , settings, wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture clearRepository(TenantId tenantId) { + ClearRepositoryGitRequest request = new ClearRepositoryGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public void processResponse(VersionControlResponseMsg vcResponseMsg) { + UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); + PendingGitRequest request = pendingRequestMap.remove(requestId); + if (request == null) { + log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); + return; + } else { + log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + request.getTimeoutTask().cancel(true); + } + var future = request.getFuture(); + if (!StringUtils.isEmpty(vcResponseMsg.getError())) { + future.setException(new RuntimeException(vcResponseMsg.getError())); + } else { + if (vcResponseMsg.hasGenericResponse()) { + future.set(null); + } else if (vcResponseMsg.hasCommitResponse()) { + var commitResponse = vcResponseMsg.getCommitResponse(); + var commitResult = new VersionCreationResult(); + if (commitResponse.getTs() > 0) { + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + } + commitResult.setAdded(commitResponse.getAdded()); + commitResult.setRemoved(commitResponse.getRemoved()); + commitResult.setModified(commitResponse.getModified()); + ((CommitGitRequest) request).getFuture().set(commitResult); + } else if (vcResponseMsg.hasListBranchesResponse()) { + var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); + } else if (vcResponseMsg.hasListEntitiesResponse()) { + var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); + ((ListEntitiesGitRequest) request).getFuture().set( + listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList())); + } else if (vcResponseMsg.hasListVersionsResponse()) { + var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); + ((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse)); + } else if (vcResponseMsg.hasEntityContentResponse()) { + var data = vcResponseMsg.getEntityContentResponse().getData(); + ((EntityContentGitRequest) request).getFuture().set(toData(data)); + } else if (vcResponseMsg.hasEntitiesContentResponse()) { + var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); + ((EntitiesContentGitRequest) request).getFuture() + .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + } else if (vcResponseMsg.hasVersionsDiffResponse()) { + TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); + List entityVersionsDiffList = diffResponse.getDiffList().stream() + .map(diff -> EntityVersionsDiff.builder() + .externalId(EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(diff.getEntityType()), + new UUID(diff.getEntityIdMSB(), diff.getEntityIdLSB()))) + .entityDataAtVersion1(StringUtils.isNotEmpty(diff.getEntityDataAtVersion1()) ? + toData(diff.getEntityDataAtVersion1()) : null) + .entityDataAtVersion2(StringUtils.isNotEmpty(diff.getEntityDataAtVersion2()) ? + toData(diff.getEntityDataAtVersion2()) : null) + .rawDiff(diff.getRawDiff()) + .build()) + .collect(Collectors.toList()); + ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); + } else if (vcResponseMsg.hasContentsDiffResponse()) { + String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); + ((ContentsDiffGitRequest) request).getFuture().set(diff); + } + } + } + + private void processTimeout(UUID requestId) { + PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + if (pendingRequest != null) { + log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout); + pendingRequest.getFuture().setException(new TimeoutException("Request timed out")); + } + } + + private PageData toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) { + var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()); + return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext()); + } + + private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName(), proto.getAuthor()); + } + + private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { + return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); + } + + @SuppressWarnings("rawtypes") + @SneakyThrows + private EntityExportData toData(String data) { + return JacksonUtil.fromString(data, EntityExportData.class); + } + + private static TbQueueCallback wrap(SettableFuture future) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static TbQueueCallback wrap(SettableFuture future, T value) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + future.set(value); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static String getRelativePath(EntityType entityType, EntityId entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private static PrepareMsg getCommitPrepareMsg(User user, VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()) + .setBranchName(request.getBranch()).setAuthorName(getAuthorName(user)).setAuthorEmail(user.getEmail()).build(); + } + + private static String getAuthorName(User user) { + List parts = new ArrayList<>(); + if (StringUtils.isNotBlank(user.getFirstName())) { + parts.add(user.getFirstName()); + } + if (StringUtils.isNotBlank(user.getLastName())) { + parts.add(user.getLastName()); + } + if (parts.isEmpty()) { + parts.add(user.getName()); + } + return String.join(" ", parts); + } + + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, RepositorySettings settings) { + var tenantId = request.getTenantId(); + var requestId = request.getRequestId(); + var builder = ToVersionControlServiceMsg.newBuilder() + .setNodeId(serviceInfoProvider.getServiceId()) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setRequestIdMSB(requestId.getMostSignificantBits()) + .setRequestIdLSB(requestId.getLeastSignificantBits()); + RepositorySettings vcSettings = settings; + if (vcSettings == null && request.requiresSettings()) { + vcSettings = entitiesVersionControlService.getVersionControlSettings(tenantId); + } + if (vcSettings != null) { + builder.setVcSettings(ByteString.copyFrom(encodingService.encode(vcSettings))); + } else if (request.requiresSettings()) { + throw new RuntimeException("No entity version control settings provisioned!"); + } + return builder; + } + + private CommitRequestMsg.Builder buildCommitRequest(CommitGitRequest commit) { + return CommitRequestMsg.newBuilder().setTxId(commit.getTxId().toString()); + } +} + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java new file mode 100644 index 0000000000..0a0f56eac6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -0,0 +1,72 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; +import java.util.UUID; + +public interface EntitiesVersionControlService { + + ListenableFuture saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception; + + ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; + + ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; + + ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; + + ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + ListenableFuture loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception; + + ListenableFuture compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; + + ListenableFuture> listBranches(TenantId tenantId) throws Exception; + + RepositorySettings getVersionControlSettings(TenantId tenantId); + + ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings); + + ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception; + + ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; + + ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception; + + ListenableFuture autoCommit(SecurityUser user, EntityType entityType, List entityIds) throws Exception; + + ListenableFuture getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId); +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java new file mode 100644 index 0000000000..a1aad04701 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -0,0 +1,75 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; + +import java.util.List; + +public interface GitVersionControlQueueService { + + ListenableFuture prepareCommit(User user, VersionCreateRequest request); + + ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); + + ListenableFuture deleteAll(CommitGitRequest pendingCommit, EntityType entityType); + + ListenableFuture push(CommitGitRequest commit); + + ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + ListenableFuture> listBranches(TenantId tenantId); + + ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); + + ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + + ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); + + ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); + + ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); + + ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); + + ListenableFuture clearRepository(TenantId tenantId); + + void processResponse(VersionControlResponseMsg vcResponseMsg); +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java new file mode 100644 index 0000000000..a1b036d37b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@SuppressWarnings("rawtypes") +public class LoadEntityException extends RuntimeException { + + private static final long serialVersionUID = -1749719992370409504L; + @Getter + private final EntityExportData data; + + public LoadEntityException(EntityExportData data, Throwable cause) { + super(cause); + this.data = data; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java new file mode 100644 index 0000000000..3a674eece0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import java.io.Serializable; + +public abstract class TbAbstractVersionControlSettingsService { + + private final String settingsKey; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + private final Class clazz; + + public TbAbstractVersionControlSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache, Class clazz, String settingsKey) { + this.adminSettingsService = adminSettingsService; + this.cache = cache; + this.clazz = clazz; + this.settingsKey = settingsKey; + } + + public T get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, settingsKey); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load " + settingsKey + " settings!", e); + } + } + return null; + }, true); + } + + public T save(TenantId tenantId, T settings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, settingsKey); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(settingsKey); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + T savedSettings; + try { + savedSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load auto commit settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedSettings; + } + + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettingsByTenantIdAndKey(tenantId, settingsKey); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java new file mode 100644 index 0000000000..47b9e5f5d2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public AutoCommitSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.AUTO_COMMIT_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java new file mode 100644 index 0000000000..f88b1cf6bd --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache { + + public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java new file mode 100644 index 0000000000..b6e8d45ec2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbAutoCommitSettingsService extends TbAbstractVersionControlSettingsService implements TbAutoCommitSettingsService { + + public static final String SETTINGS_KEY = "autoCommitSettings"; + + public DefaultTbAutoCommitSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, AutoCommitSettings.class, SETTINGS_KEY); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java new file mode 100644 index 0000000000..51978481e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbAutoCommitSettingsService { + + AutoCommitSettings get(TenantId tenantId); + + AutoCommitSettings save(TenantId tenantId, AutoCommitSettings settings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java new file mode 100644 index 0000000000..09245eb895 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; + +public class ClearRepositoryGitRequest extends VoidGitRequest { + + public ClearRepositoryGitRequest(TenantId tenantId) { + super(tenantId); + } + + public boolean requiresSettings() { + return false; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java new file mode 100644 index 0000000000..a989439d10 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +public class CommitGitRequest extends PendingGitRequest { + + @Getter + private final UUID txId; + private final VersionCreateRequest request; + + public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) { + super(tenantId); + this.txId = UUID.randomUUID(); + this.request = request; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java new file mode 100644 index 0000000000..77482fdf71 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.HashMap; +import java.util.Map; + +public class ComplexEntitiesExportCtx extends EntitiesExportCtx { + + private final Map settings = new HashMap<>(); + + public ComplexEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, ComplexVersionCreateRequest request) { + super(user, commit, request); + request.getEntityTypes().forEach((type, config) -> settings.put(type, buildExportSettings(config))); + } + + public EntityExportSettings getSettings(EntityType entityType) { + return settings.get(entityType); + } + + @Override + public EntityExportSettings getSettings() { + throw new RuntimeException("Not implemented. Use EntityTypeExportCtx instead!"); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java new file mode 100644 index 0000000000..7c3fb7a600 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class ContentsDiffGitRequest extends PendingGitRequest { + + private final String content1; + private final String content2; + + public ContentsDiffGitRequest(TenantId tenantId, String content1, String content2) { + super(tenantId); + this.content1 = content1; + this.content2 = content2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java new file mode 100644 index 0000000000..ad653edd0c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +import java.util.List; + +@Getter +public class EntitiesContentGitRequest extends PendingGitRequest> { + + private final String versionId; + private final EntityType entityType; + + public EntitiesContentGitRequest(TenantId tenantId, String versionId, EntityType entityType) { + super(tenantId); + this.versionId = versionId; + this.entityType = entityType; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java new file mode 100644 index 0000000000..2ea3b18e5c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Data +public abstract class EntitiesExportCtx { + + protected final SecurityUser user; + protected final CommitGitRequest commit; + protected final R request; + private final List> futures; + private final Map externalIdMap; + + public EntitiesExportCtx(SecurityUser user, CommitGitRequest commit, R request) { + this.user = user; + this.commit = commit; + this.request = request; + this.futures = new ArrayList<>(); + this.externalIdMap = new HashMap<>(); + } + + protected EntitiesExportCtx(EntitiesExportCtx other) { + this.user = other.getUser(); + this.commit = other.getCommit(); + this.request = other.getRequest(); + this.futures = other.getFutures(); + this.externalIdMap = other.getExternalIdMap(); + } + + public void add(ListenableFuture future) { + futures.add(future); + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + protected static EntityExportSettings buildExportSettings(VersionCreateConfig config) { + return EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + } + + public abstract EntityExportSettings getSettings(); + + @SuppressWarnings("unchecked") + public ID getExternalId(ID internalId) { + var result = externalIdMap.get(internalId); + log.debug("[{}][{}] Local cache {} for id", internalId.getEntityType(), internalId.getId(), result != null ? "hit" : "miss"); + return (ID) result; + } + + public void putExternalId(EntityId internalId, EntityId externalId) { + log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); + externalIdMap.put(internalId, externalId != null ? externalId : internalId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java new file mode 100644 index 0000000000..c781f3cd18 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -0,0 +1,141 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Slf4j +@Data +public class EntitiesImportCtx { + + private final SecurityUser user; + private final String versionId; + + private final Map results = new HashMap<>(); + private final Map> importedEntities = new HashMap<>(); + private final Map toReimport = new HashMap<>(); + private final List referenceCallbacks = new ArrayList<>(); + private final List eventCallbacks = new ArrayList<>(); + private final Map externalToInternalIdMap = new HashMap<>(); + private final Set notFoundIds = new HashSet<>(); + + private final Set relations = new LinkedHashSet<>(); + + private boolean finalImportAttempt = false; + private EntityImportSettings settings; + private EntityImportResult currentImportResult; + + public EntitiesImportCtx(SecurityUser user, String versionId) { + this(user, versionId, null); + } + + public EntitiesImportCtx(SecurityUser user, String versionId, EntityImportSettings settings) { + this.user = user; + this.versionId = versionId; + this.settings = settings; + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + public boolean isFindExistingByName() { + return getSettings().isFindExistingByName(); + } + + public boolean isUpdateRelations() { + return getSettings().isUpdateRelations(); + } + + public boolean isSaveAttributes() { + return getSettings().isSaveAttributes(); + } + + public boolean isSaveCredentials() { + return getSettings().isSaveCredentials(); + } + + public EntityId getInternalId(EntityId externalId) { + var result = externalToInternalIdMap.get(externalId); + log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); + return result; + } + + public void putInternalId(EntityId externalId, EntityId internalId) { + log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId); + externalToInternalIdMap.put(externalId, internalId); + } + + public void registerResult(EntityType entityType, boolean created) { + EntityTypeLoadResult result = results.computeIfAbsent(entityType, EntityTypeLoadResult::new); + if (created) { + result.setCreated(result.getCreated() + 1); + } else { + result.setUpdated(result.getUpdated() + 1); + } + } + + public void registerDeleted(EntityType entityType) { + EntityTypeLoadResult result = results.computeIfAbsent(entityType, EntityTypeLoadResult::new); + result.setDeleted(result.getDeleted() + 1); + } + + public void addRelations(Collection values) { + relations.addAll(values); + } + + public void addReferenceCallback(ThrowingRunnable tr) { + if (tr != null) { + referenceCallbacks.add(tr); + } + } + + public void addEventCallback(ThrowingRunnable tr) { + if (tr != null) { + eventCallbacks.add(tr); + } + } + + public void registerNotFound(EntityId externalId) { + notFoundIds.add(externalId); + } + + public boolean isNotFound(EntityId externalId) { + return notFoundIds.contains(externalId); + } + + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java new file mode 100644 index 0000000000..6a1d60a065 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Getter +public class EntityContentGitRequest extends PendingGitRequest { + + private final String versionId; + private final EntityId entityId; + + public EntityContentGitRequest(TenantId tenantId, String versionId, EntityId entityId) { + super(tenantId); + this.versionId = versionId; + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java new file mode 100644 index 0000000000..80d90590a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.apache.commons.lang3.ObjectUtils; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +public class EntityTypeExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityType entityType; + @Getter + private final boolean overwrite; + @Getter + private final EntityExportSettings settings; + + public EntityTypeExportCtx(EntitiesExportCtx parent, EntityTypeVersionCreateConfig config, SyncStrategy defaultSyncStrategy, EntityType entityType) { + super(parent); + this.entityType = entityType; + this.settings = EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java new file mode 100644 index 0000000000..c045030dc7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; + +public class ListBranchesGitRequest extends PendingGitRequest> { + + public ListBranchesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java new file mode 100644 index 0000000000..3fc531735e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListEntitiesGitRequest extends PendingGitRequest> { + + public ListEntitiesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java new file mode 100644 index 0000000000..d40a589646 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; + +public class ListVersionsGitRequest extends PendingGitRequest> { + + public ListVersionsGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java new file mode 100644 index 0000000000..b34806a6ca --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import com.google.common.util.concurrent.SettableFuture; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; +import java.util.concurrent.ScheduledFuture; + +@Getter +public class PendingGitRequest { + + private final long createdTime; + private final UUID requestId; + private final TenantId tenantId; + private final SettableFuture future; + @Setter + private ScheduledFuture timeoutTask; + + public PendingGitRequest(TenantId tenantId) { + this.createdTime = System.currentTimeMillis(); + this.requestId = UUID.randomUUID(); + this.tenantId = tenantId; + this.future = SettableFuture.create(); + } + + public boolean requiresSettings() { + return true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java new file mode 100644 index 0000000000..97432adbb8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; + +@Data +public class ReimportTask { + + private final EntityExportData data; + private final EntityImportSettings settings; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java new file mode 100644 index 0000000000..8cb9cc555e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +public class SimpleEntitiesExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityExportSettings settings; + + public SimpleEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, SingleEntityVersionCreateRequest request) { + this(user, commit, request, request != null ? buildExportSettings(request.getConfig()) : null); + } + + public SimpleEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, SingleEntityVersionCreateRequest request, EntityExportSettings settings) { + super(user, commit, request); + this.settings = settings; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java new file mode 100644 index 0000000000..e73706609f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; + +import java.util.List; + +@Getter +public class VersionsDiffGitRequest extends PendingGitRequest> { + + private final String path; + private final String versionId1; + private final String versionId2; + + public VersionsDiffGitRequest(TenantId tenantId, String path, String versionId1, String versionId2) { + super(tenantId); + this.path = path; + this.versionId1 = versionId1; + this.versionId2 = versionId2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java new file mode 100644 index 0000000000..c48bfa7a03 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; + +public class VoidGitRequest extends PendingGitRequest { + + public VoidGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java new file mode 100644 index 0000000000..555490bd06 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbRepositorySettingsService extends TbAbstractVersionControlSettingsService implements TbRepositorySettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + + public DefaultTbRepositorySettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, RepositorySettings.class, SETTINGS_KEY); + } + + @Override + public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { + RepositorySettings storedSettings = get(tenantId); + if (storedSettings != null) { + RepositoryAuthMethod authMethod = settings.getAuthMethod(); + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public RepositorySettings get(TenantId tenantId) { + RepositorySettings settings = super.get(tenantId); + if (settings != null) { + settings = new RepositorySettings(settings); + } + return settings; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java new file mode 100644 index 0000000000..60b7f50e12 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("RepositorySettingsCache") +public class RepositorySettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public RepositorySettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.REPOSITORY_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java new file mode 100644 index 0000000000..3cccb6d24e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("RepositorySettingsCache") +public class RepositorySettingsRedisCache extends RedisTbTransactionalCache { + + public RepositorySettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java new file mode 100644 index 0000000000..946d06c87c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbRepositorySettingsService { + + RepositorySettings restore(TenantId tenantId, RepositorySettings versionControlSettings); + + RepositorySettings get(TenantId tenantId); + + RepositorySettings save(TenantId tenantId, RepositorySettings versionControlSettings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 6c4adfc5ae..f8b42ecc1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -65,7 +65,7 @@ import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProvisionService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java index 4e3abd8e68..e178055706 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java @@ -32,7 +32,7 @@ import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.action.EntityActionService; @@ -50,7 +50,7 @@ public class AlarmsCleanUpService { @Value("${sql.ttl.alarms.removal_batch_size}") private Integer removalBatchSize; - private final TenantDao tenantDao; + private final TenantService tenantService; private final AlarmDao alarmDao; private final AlarmService alarmService; private final RelationService relationService; @@ -64,7 +64,7 @@ public class AlarmsCleanUpService { PageLink removalBatchRequest = new PageLink(removalBatchSize, 0 ); PageData tenantsIds; do { - tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest); + tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); for (TenantId tenantId : tenantsIds.getData()) { if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { continue; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index 877987d6f0..1c21cc6029 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -43,7 +43,7 @@ public class RpcCleanUpService { @Value("${sql.ttl.rpc.enabled}") private boolean ttlTaskExecutionEnabled; - private final TenantDao tenantDao; + private final TenantService tenantService; private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; private final RpcDao rpcDao; @@ -54,7 +54,7 @@ public class RpcCleanUpService { PageLink tenantsBatchRequest = new PageLink(10_000, 0); PageData tenantsIds; do { - tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest); + tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); for (TenantId tenantId : tenantsIds.getData()) { if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { continue; diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index c68d484cd5..995ebb43c4 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -26,6 +26,9 @@ + + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9bb0e8a9ff..72ed590c4f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -394,6 +394,9 @@ cache: tenantProfiles: timeToLiveInMinutes: "${CACHE_SPECS_TENANT_PROFILES_TTL:1440}" maxSize: "${CACHE_SPECS_TENANT_PROFILES_MAX_SIZE:10000}" + tenants: + timeToLiveInMinutes: "${CACHE_SPECS_TENANTS_TTL:1440}" + maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" deviceProfiles: timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_PROFILES_TTL:1440}" maxSize: "${CACHE_SPECS_DEVICE_PROFILES_MAX_SIZE:10000}" @@ -412,6 +415,12 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" + repositorySettings: + timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" + autoCommitSettings: + timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" twoFaVerificationCodes: timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}" maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}" @@ -910,6 +919,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" @@ -1003,6 +1015,12 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1096,6 +1114,14 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + management: endpoints: web: diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ee2ba371..80eaed9ff1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -656,7 +656,6 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected T readResponse(ResultActions result, TypeReference type) throws Exception { byte[] content = result.andReturn().getResponse().getContentAsByteArray(); - ObjectMapper mapper = new ObjectMapper(); return mapper.readerFor(type).readValue(content); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java index 1f94fa4784..45c8d72529 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -108,7 +109,7 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doPost("/api/customer", savedCustomer, Customer.class); testNotifyEntityAllOneTime(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), savedCustomer.getTenantId(), - savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + new CustomerId(CustomerId.NULL_UUID), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java index cea08003bf..9df6c16111 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java @@ -71,6 +71,8 @@ public class HashPartitionServiceTest { queueRoutingInfoService); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10); + ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); + ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() .setServiceId("tb-core-0") diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java new file mode 100644 index 0000000000..9b34eaae25 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -0,0 +1,451 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; + +public abstract class BaseExportImportServiceTest extends AbstractControllerTest { + + @Autowired + protected EntitiesExportImportService exportImportService; + @Autowired + protected DeviceService deviceService; + @Autowired + protected OtaPackageService otaPackageService; + @Autowired + protected DeviceProfileService deviceProfileService; + @Autowired + protected AssetService assetService; + @Autowired + protected CustomerService customerService; + @Autowired + protected RuleChainService ruleChainService; + @Autowired + protected DashboardService dashboardService; + @Autowired + protected RelationService relationService; + @Autowired + protected TenantService tenantService; + @Autowired + protected EntityViewService entityViewService; + + protected TenantId tenantId1; + protected User tenantAdmin1; + + protected TenantId tenantId2; + protected User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } + + protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName(name); + device.setLabel("lbl"); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + device.setDeviceData(deviceData); + return deviceService.saveDevice(device); + } + + protected OtaPackage createOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type) { + OtaPackage otaPackage = new OtaPackage(); + otaPackage.setTenantId(tenantId); + otaPackage.setDeviceProfileId(deviceProfileId); + otaPackage.setType(type); + otaPackage.setTitle("My " + type); + otaPackage.setVersion("v1.0"); + otaPackage.setFileName("filename.txt"); + otaPackage.setContentType("text/plain"); + otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); + otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); + otaPackage.setDataSize(1L); + otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1})); + return otaPackageService.saveOtaPackage(otaPackage); + } + + protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, RuleChainId defaultRuleChainId, DashboardId defaultDashboardId, String name) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setName(name); + deviceProfile.setDescription("dscrptn"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + deviceProfile.setDefaultRuleChainId(defaultRuleChainId); + deviceProfile.setDefaultDashboardId(defaultDashboardId); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + protected void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setCustomerId(customerId); + asset.setType(type); + asset.setName(name); + asset.setLabel("lbl"); + asset.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return assetService.saveAsset(asset); + } + + protected void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + + protected Customer createCustomer(TenantId tenantId, String name) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(name); + customer.setCountry("ua"); + customer.setAddress("abb"); + customer.setEmail("ccc@aa.org"); + customer.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return customerService.saveCustomer(customer); + } + + protected void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenantId); + dashboard.setTitle(name); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + dashboard.setImage("abvregewrg"); + dashboard.setMobileHide(true); + dashboard = dashboardService.saveDashboard(dashboard); + if (customerId != null) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + return dashboardService.findDashboardById(tenantId, dashboard.getId()); + } + return dashboard; + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name, AssetId assetForEntityAlias) { + Dashboard dashboard = createDashboard(tenantId, customerId, name); + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + assetForEntityAlias.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + return dashboardService.saveDashboard(dashboard); + } + + protected void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + protected RuleChain createRuleChain(TenantId tenantId, String name, EntityId originatorId) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Generator 1"); + ruleNode1.setType(TbMsgGeneratorNode.class.getName()); + ruleNode1.setDebugMode(true); + TbMsgGeneratorNodeConfiguration configuration1 = new TbMsgGeneratorNodeConfiguration(); + configuration1.setOriginatorType(originatorId.getEntityType()); + configuration1.setOriginatorId(originatorId.getId().toString()); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + + protected RuleChain createRuleChain(TenantId tenantId, String name) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + protected EntityView createEntityView(TenantId tenantId, CustomerId customerId, EntityId entityId, String name) { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setEntityId(entityId); + entityView.setCustomerId(customerId); + entityView.setName(name); + entityView.setType("A"); + return entityViewService.saveEntityView(entityView); + } + + protected EntityRelation createRelation(EntityId from, EntityId to) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(from); + relation.setTo(to); + relation.setType(EntityRelation.MANAGES_TYPE); + relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(TenantId.SYS_TENANT_ID, relation); + return relation; + } + + protected & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + } + } + + + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { + return exportEntity(user, entityId, EntityExportSettings.builder() + .exportCredentials(true) + .build()); + } + + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId, EntityExportSettings exportSettings) throws Exception { + return exportImportService.exportEntity(new SimpleEntitiesExportCtx(getSecurityUser(user), null, null, exportSettings), entityId); + } + + protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData) throws Exception { + return importEntity(user, exportData, EntityImportSettings.builder() + .saveCredentials(true) + .build()); + } + + protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData, EntityImportSettings importSettings) throws Exception { + EntitiesImportCtx ctx = new EntitiesImportCtx(getSecurityUser(user), null, importSettings); + ctx.setFinalImportAttempt(true); + exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class); + EntityImportResult importResult = exportImportService.importEntity(ctx, exportData); + exportImportService.saveReferencesAndRelations(ctx); + for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { + throwingRunnable.run(); + } + return importResult; + } + + protected SecurityUser getSecurityUser(User user) { + return new SecurityUser(user, true, new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail())); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java new file mode 100644 index 0000000000..3e3a6d5cae --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -0,0 +1,585 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.Streams; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceProfileDao; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.ota.OtaPackageStateService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.verify; + +@DaoSqlTest +public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { + + @Autowired + private DeviceCredentialsService deviceCredentialsService; + @SpyBean + private EntityActionService entityActionService; + @SpyBean + private OtaPackageStateService otaPackageStateService; + + @Test + public void testExportImportAsset_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, asset.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, asset, tenantId2, importResult.getSavedEntity()); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_sameTenant() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, asset.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, asset, tenantId1, importResult.getSavedEntity()); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_sameTenant_withCustomer() throws Exception { + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + Asset importedAsset = importEntity(tenantAdmin1, this.exportEntity(tenantAdmin1, asset.getId())).getSavedEntity(); + assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); + } + + + @Test + public void testExportImportCustomer_betweenTenants() throws Exception { + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, customer.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, customer, tenantId2, importResult.getSavedEntity()); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + @Test + public void testExportImportCustomer_sameTenant() throws Exception { + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, customer.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, customer, tenantId1, importResult.getSavedEntity()); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + + @Test + public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile of tenant 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData profileExportData = exportEntity(tenantAdmin1, deviceProfile.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId()); + DeviceCredentials exportedCredentials = ((DeviceExportData) deviceExportData).getCredentials(); + exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); + + EntityImportResult profileImportResult = importEntity(tenantAdmin2, profileExportData); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult.getSavedEntity()); + checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); + + EntityImportResult deviceImportResult = importEntity(tenantAdmin2, deviceExportData); + Device importedDevice = deviceImportResult.getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult.getSavedEntity()); + checkImportedDeviceData(device, importedDevice); + + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getId()).isNotEqualTo(credentials.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(exportedCredentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + } + + @Test + public void testExportImportDevice_sameTenant() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile v1.0"); + OtaPackage firmware = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.FIRMWARE); + OtaPackage software = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.SOFTWARE); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + device.setFirmwareId(firmware.getId()); + device.setSoftwareId(software.getId()); + device = deviceService.saveDevice(device); + + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, deviceExportData); + Device importedDevice = importResult.getSavedEntity(); + + checkImportedEntity(tenantId1, device, tenantId1, importResult.getSavedEntity()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); + assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + assertThat(importedDevice.getFirmwareId()).isEqualTo(firmware.getId()); + assertThat(importedDevice.getSoftwareId()).isEqualTo(software.getId()); + } + + + @Test + public void testExportImportDashboard_betweenTenants() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } + + @Test + public void testExportImportDashboard_sameTenant() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } + + @Test + public void testExportImportDashboard_betweenTenants_withCustomer_updated() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Dashboard importedDashboard = importEntity(tenantAdmin2, exportData).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + EntityExportData customerExportData = exportEntity(tenantAdmin1, customer.getId()); + dashboardService.assignDashboardToCustomer(tenantId1, dashboard.getId(), customer.getId()); + exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Customer importedCustomer = importEntity(tenantAdmin2, customerExportData).getSavedEntity(); + importedDashboard = importEntity(tenantAdmin2, exportData).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } + + @Test + public void testExportImportDashboard_betweenTenants_withEntityAliases() throws Exception { + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + asset1.getId().toString() + "\",\n" + + "\t\t\t\t\"" + asset2.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + dashboard = dashboardService.saveDashboard(dashboard); + + EntityExportData asset1ExportData = exportEntity(tenantAdmin1, asset1.getId()); + EntityExportData asset2ExportData = exportEntity(tenantAdmin1, asset2.getId()); + EntityExportData dashboardExportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Asset importedAsset1 = importEntity(tenantAdmin2, asset1ExportData).getSavedEntity(); + Asset importedAsset2 = importEntity(tenantAdmin2, asset2ExportData).getSavedEntity(); + Dashboard importedDashboard = importEntity(tenantAdmin2, dashboardExportData).getSavedEntity(); + + Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .map(JsonNode::asText).collect(Collectors.toSet()); + assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); + assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); + } + + + @Test + public void testExportImportRuleChain_betweenTenants() throws Exception { + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + EntityExportData exportData = exportEntity(tenantAdmin1, ruleChain.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult.getSavedEntity()); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + @Test + public void testExportImportRuleChain_sameTenant() throws Exception { + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + EntityExportData exportData = exportEntity(tenantAdmin1, ruleChain.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult.getSavedEntity()); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + + @Test + public void testExportImportWithInboundRelations_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId()); + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .exportCredentials(false) + .build()); + + assertThat(deviceExportData.getRelations()).size().isOne(); + assertThat(deviceExportData.getRelations().get(0)).matches(entityRelation -> { + return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); + }); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); + + Asset importedAsset = importEntity(tenantAdmin2, assetExportData).getSavedEntity(); + Device importedDevice = importEntity(tenantAdmin2, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); + } + + @Test + public void testExportImportWithRelations_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId()); + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .exportCredentials(false) + .build()); + deviceExportData.getEntity().setDeviceProfileId(null); + + Asset importedAsset = importEntity(tenantAdmin2, assetExportData).getSavedEntity(); + Device importedDevice = importEntity(tenantAdmin2, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()).getSavedEntity(); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); + } + + @Test + public void testExportImportWithRelations_sameTenant() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device1 = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId(), EntityExportSettings.builder() + .exportRelations(true) + .build()); + assertThat(assetExportData.getRelations()).size().isOne(); + + Device device2 = createDevice(tenantId1, null, null, "Device 2"); + EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); + + importEntity(tenantAdmin1, assetExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()); + + List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + @Test + public void textExportImportWithRelations_sameTenant_removeExisting() throws Exception { + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .build()); + assertThat(deviceExportData.getRelations()).size().isOne(); + + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); + + importEntity(tenantAdmin1, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()); + + List relations = relationService.findByTo(TenantId.SYS_TENANT_ID, device.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + + @Test + public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception { + DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); + EntityExportData deviceProfileExportData = exportEntity(tenantAdmin1, defaultDeviceProfile.getId()); + + assertThatThrownBy(() -> { + importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() + .findExistingByName(false) + .build()); + }).hasMessageContaining("default device profile is present"); + + importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() + .findExistingByName(true) + .build()); + checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); + } + + + @SuppressWarnings("rawTypes") + private static EntityExportData getAndClone(Map map, EntityType entityType) { + return JacksonUtil.clone(map.get(entityType)); + } + + @SuppressWarnings({"rawTypes", "unchecked"}) + @Test + public void testEntityEventsOnImport() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + + Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), + ruleChain.getId(), dashboard.getId(), deviceProfile.getId()) + .map(entityId -> { + try { + return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() + .exportCredentials(false) + .build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toMap(EntityExportData::getEntityType, d -> d)); + + Mockito.reset(entityActionService); + Customer importedCustomer = (Customer) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CUSTOMER)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.ADDED), isNull()); + Mockito.reset(entityActionService); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CUSTOMER)); + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + + EntityExportData updatedCustomerEntity = getAndClone(entitiesExportData, EntityType.CUSTOMER); + updatedCustomerEntity.getEntity().setEmail("t" + updatedCustomerEntity.getEntity().getEmail()); + Customer updatedCustomer = importEntity(tenantAdmin2, updatedCustomerEntity).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(updatedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + Mockito.reset(entityActionService); + + Asset importedAsset = (Asset) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.ASSET)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.ADDED), isNull()); + importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)); + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.UPDATED), isNull()); + + + EntityExportData updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); + updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); + Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); + + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(updatedAsset), + any(), eq(ActionType.UPDATED), isNull()); + verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + RuleChain importedRuleChain = (RuleChain) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.RULE_CHAIN)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), + any(), eq(ActionType.ADDED), isNull()); + verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); + + Dashboard importedDashboard = (Dashboard) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DASHBOARD)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), + any(), eq(ActionType.ADDED), isNull()); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE_PROFILE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), + any(), eq(ActionType.ADDED), isNull()); + verify(tbClusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); + verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(tbClusterService).sendNotificationMsgToEdgeService(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); + verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); + + Device importedDevice = (Device) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), + any(), eq(ActionType.ADDED), isNull()); + verify(tbClusterService).onDeviceUpdated(eq(importedDevice), isNull()); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); + verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + + EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); + updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); + Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); + verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); + } + + @Test + public void testExternalIdsInExportData() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, customer.getId(), "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1", asset.getId()); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1", asset.getId()); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); + EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); + + Map ids = new HashMap<>(); + for (EntityId entityId : List.of(customer.getId(), asset.getId(), ruleChain.getId(), dashboard.getId(), + deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { + EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); + EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() + .saveCredentials(false) + .build()); + ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); + } + + Asset exportedAsset = (Asset) exportEntity(tenantAdmin2, (AssetId) ids.get(asset.getId())).getEntity(); + assertThat(exportedAsset.getCustomerId()).isEqualTo(customer.getId()); + + EntityExportData ruleChainExportData = exportEntity(tenantAdmin2, (RuleChainId) ids.get(ruleChain.getId())); + TbMsgGeneratorNodeConfiguration exportedRuleNodeConfig = ((RuleChainExportData) ruleChainExportData).getMetaData().getNodes().stream() + .filter(node -> node.getType().equals(TbMsgGeneratorNode.class.getName())).findFirst() + .map(RuleNode::getConfiguration).map(config -> JacksonUtil.treeToValue(config, TbMsgGeneratorNodeConfiguration.class)).orElse(null); + assertThat(exportedRuleNodeConfig.getOriginatorId()).isEqualTo(asset.getId().toString()); + + Dashboard exportedDashboard = (Dashboard) exportEntity(tenantAdmin2, (DashboardId) ids.get(dashboard.getId())).getEntity(); + assertThat(exportedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(shortCustomerInfo -> { + assertThat(shortCustomerInfo.getCustomerId()).isEqualTo(customer.getId()); + }); + String exportedEntityAliasAssetId = exportedDashboard.getConfiguration().get("entityAliases").elements().next() + .get("filter").get("entityList").elements().next().asText(); + assertThat(exportedEntityAliasAssetId).isEqualTo(asset.getId().toString()); + + DeviceProfile exportedDeviceProfile = (DeviceProfile) exportEntity(tenantAdmin2, (DeviceProfileId) ids.get(deviceProfile.getId())).getEntity(); + assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); + assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); + + Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); + assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); + assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); + + EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); + assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); + assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); + + deviceProfile.setDefaultDashboardId(null); + deviceProfileService.saveDeviceProfile(deviceProfile); + DeviceProfile importedDeviceProfile = deviceProfileService.findDeviceProfileById(tenantId2, (DeviceProfileId) ids.get(deviceProfile.getId())); + importedDeviceProfile.setDefaultDashboardId(null); + deviceProfileService.saveDeviceProfile(importedDeviceProfile); + } + +} diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 22953f259a..e2486f3a7e 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -31,8 +31,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueClusterService; @@ -47,9 +48,11 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 50f114856f..3079978bce 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -618,8 +618,8 @@ message TbSubscriptionUpdateValueListProto { } message TbSubscriptionUpdateTsValue { - int64 ts = 1; - optional string value = 2; + int64 ts = 1; + optional string value = 2; } /** @@ -676,6 +676,188 @@ message EdgeNotificationMsgProto { PostAttributeMsg postAttributesMsg = 12; } +/** + TB Core to Version Control Service + */ +message CommitRequestMsg { + string txId = 1; // To correlate prepare, add, delete and push messages + PrepareMsg prepareMsg = 2; + AddMsg addMsg = 3; + DeleteMsg deleteMsg = 4; + PushMsg pushMsg = 5; + AbortMsg abortMsg = 6; +} + +message CommitResponseMsg { + int64 ts = 1; + string commitId = 2; + string name = 3; + string author = 4; + int32 added = 5; + int32 modified = 6; + int32 removed = 7; +} + +message PrepareMsg { + string commitMsg = 1; + string branchName = 2; + string authorName = 3; + string authorEmail = 4; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} + +message ListVersionsRequestMsg { + string branchName = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; + int32 pageSize = 5; + int32 page = 6; + string textSearch = 7; + string sortProperty = 8; + string sortDirection = 9; +} + +message EntityVersionProto { + int64 ts = 1; + string id = 2; + string name = 3; + string author = 4; +} + +message ListVersionsResponseMsg { + repeated EntityVersionProto versions = 1; + int32 totalPages = 2; + int64 totalElements = 3; + bool hasNext = 4; +} + +message ListEntitiesRequestMsg { + string branchName = 1; + string versionId = 2; + string entityType = 3; +} + +message VersionedEntityInfoProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; +} + +message ListEntitiesResponseMsg { + repeated VersionedEntityInfoProto entities = 1; +} + +message ListBranchesRequestMsg { +} + +message ListBranchesResponseMsg { + repeated string branches = 1; +} + +message EntityContentRequestMsg { + string versionId = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message EntityContentResponseMsg { + string data = 1; +} + +message EntitiesContentRequestMsg { + string versionId = 1; + string entityType = 2; + int32 offset = 3; + int32 limit = 4; +} + +message EntitiesContentResponseMsg { + repeated string data = 1; +} + +message VersionsDiffRequestMsg { + string path = 1; + string versionId1 = 2; + string versionId2 = 3; +} + +message VersionsDiffResponseMsg { + repeated EntityVersionsDiff diff = 1; +} + +message EntityVersionsDiff { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + string entityDataAtVersion1 = 4; + string entityDataAtVersion2 = 5; + string rawDiff = 6; +} + +message ContentsDiffRequestMsg { + string content1 = 1; + string content2 = 2; +} + +message ContentsDiffResponseMsg { + string diff = 1; +} + +message GenericRepositoryRequestMsg {} + +message GenericRepositoryResponseMsg {} + +message ToVersionControlServiceMsg { + string nodeId = 1; + int64 tenantIdMSB = 2; + int64 tenantIdLSB = 3; + int64 requestIdMSB = 4; + int64 requestIdLSB = 5; + bytes vcSettings = 6; + GenericRepositoryRequestMsg initRepositoryRequest = 7; + GenericRepositoryRequestMsg testRepositoryRequest = 8; + GenericRepositoryRequestMsg clearRepositoryRequest = 9; + CommitRequestMsg commitRequest = 10; + ListVersionsRequestMsg listVersionRequest = 11; + ListEntitiesRequestMsg listEntitiesRequest = 12; + ListBranchesRequestMsg listBranchesRequest = 13; + EntityContentRequestMsg entityContentRequest = 14; + EntitiesContentRequestMsg entitiesContentRequest = 15; + VersionsDiffRequestMsg versionsDiffRequest = 16; + ContentsDiffRequestMsg contentsDiffRequest = 17; +} + +message VersionControlResponseMsg { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string error = 3; + GenericRepositoryResponseMsg genericResponse = 4; + CommitResponseMsg commitResponse = 5; + ListBranchesResponseMsg listBranchesResponse = 6; + ListEntitiesResponseMsg listEntitiesResponse = 7; + ListVersionsResponseMsg listVersionsResponse = 8; + EntityContentResponseMsg entityContentResponse = 9; + EntitiesContentResponseMsg entitiesContentResponse = 10; + VersionsDiffResponseMsg versionsDiffResponse = 11; + ContentsDiffResponseMsg contentsDiffResponse = 12; +} + /** * Main messages; */ @@ -730,6 +912,7 @@ message ToCoreNotificationMsg { bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; + VersionControlResponseMsg vcResponseMsg = 7; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -793,3 +976,5 @@ message ToOtaPackageStateServiceMsg { int64 otaPackageIdLSB = 7; string type = 8; } + + diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java index 84c3f9332f..4032797b5b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import java.util.List; + public interface DashboardService { Dashboard findDashboardById(TenantId tenantId, DashboardId dashboardId); @@ -64,4 +66,7 @@ public interface DashboardService { PageData findDashboardsByTenantIdAndEdgeId(TenantId tenantId, EdgeId edgeId, PageLink pageLink); DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name); + + List findTenantDashboardsByTitle(TenantId tenantId, String title); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java index bbc011845a..36343c3662 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java @@ -72,6 +72,8 @@ public interface EntityViewService { ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId); + List findEntityViewsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + void deleteEntityView(TenantId tenantId, EntityViewId entityViewId); void deleteEntityViewsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index dbd641c1c8..e7df5eea93 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import java.util.Collection; import java.util.List; /** @@ -34,12 +35,16 @@ import java.util.List; */ public interface RelationService { - ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + + boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean saveRelation(TenantId tenantId, EntityRelation relation); + void saveRelations(TenantId tenantId, List relations); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); boolean deleteRelation(TenantId tenantId, EntityRelation relation); @@ -52,8 +57,6 @@ public interface RelationService { void deleteEntityRelations(TenantId tenantId, EntityId entity); - ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entity); - List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); ListenableFuture> findByFromAsync(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 3328372f2e..9a856ae66a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -28,11 +28,11 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainData; import org.thingsboard.server.common.data.rule.RuleChainImportResult; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.data.rule.RuleNode; +import java.util.Collection; import java.util.List; /** @@ -66,6 +66,8 @@ public interface RuleChainService { PageData findTenantRuleChainsByType(TenantId tenantId, RuleChainType type, PageLink pageLink); + Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name); + void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId); void deleteRuleChainsByTenantId(TenantId tenantId); @@ -97,4 +99,7 @@ public interface RuleChainService { PageData findAllRuleNodesByType(String type, PageLink pageLink); RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode); + + void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java index d9238c9d7b..82839da1b8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java @@ -25,6 +25,12 @@ public interface AdminSettingsService { AdminSettings findAdminSettingsByKey(TenantId tenantId, String key); + AdminSettings findAdminSettingsByTenantIdAndKey(TenantId tenantId, String key); + AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings); + boolean deleteAdminSettingsByTenantIdAndKey(TenantId tenantId, String key); + + void deleteAdminSettingsByTenantId(TenantId tenantId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java index fbb9dfa0e1..2dcccc843f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java @@ -35,6 +35,8 @@ public interface TenantService { Tenant saveTenant(Tenant tenant); + boolean tenantExists(TenantId tenantId); + void deleteTenant(TenantId tenantId); PageData findTenants(PageLink pageLink); @@ -44,4 +46,6 @@ public interface TenantService { List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId); void deleteTenants(); + + PageData findTenantsIds(PageLink pageLink); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index 3b639659c8..d88fabc052 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.AdminSettingsId; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class AdminSettings extends BaseData { +public class AdminSettings extends BaseData implements HasTenantId { private static final long serialVersionUID = -7670322981725511892L; + private TenantId tenantId; + @NoXss @Length(fieldName = "key") private String key; @@ -44,6 +47,7 @@ public class AdminSettings extends BaseData { public AdminSettings(AdminSettings adminSettings) { super(adminSettings); + this.tenantId = adminSettings.getTenantId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -60,7 +64,16 @@ public class AdminSettings extends BaseData { return super.getCreatedTime(); } - @ApiModelProperty(position = 3, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") + @ApiModelProperty(position = 3, value = "JSON object with Tenant Id.", readOnly = true) + public TenantId getTenantId() { + return tenantId; + } + + public void setTenantId(TenantId tenantId) { + this.tenantId = tenantId; + } + + @ApiModelProperty(position = 4, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") public String getKey() { return key; } @@ -69,7 +82,7 @@ public class AdminSettings extends BaseData { this.key = key; } - @ApiModelProperty(position = 4, value = "JSON representation of the Administration Settings value") + @ApiModelProperty(position = 5, value = "JSON representation of the Administration Settings value") public JsonNode getJsonValue() { return jsonValue; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java index c8cf730ead..f2ada352e7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java @@ -65,9 +65,7 @@ public abstract class BaseData extends IdBased implement if (getClass() != obj.getClass()) return false; BaseData other = (BaseData) obj; - if (createdTime != other.createdTime) - return false; - return true; + return createdTime == other.createdTime; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 49db7befd3..ee0d9ee119 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -26,10 +26,13 @@ public class CacheConstants { public static final String CLAIM_DEVICES_CACHE = "claimDevices"; public static final String SECURITY_SETTINGS_CACHE = "securitySettings"; public static final String TENANT_PROFILE_CACHE = "tenantProfiles"; + public static final String TENANTS_CACHE = "tenants"; public static final String DEVICE_PROFILE_CACHE = "deviceProfiles"; public static final String ATTRIBUTES_CACHE = "attributes"; public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; + public static final String REPOSITORY_SETTINGS_CACHE = "repositorySettings"; + public static final String AUTO_COMMIT_SETTINGS_CACHE = "autoCommitSettings"; public static final String TWO_FA_VERIFICATION_CODES_CACHE = "twoFaVerificationCodes"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index ab21d70795..9ce638e667 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,12 +20,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -public class Customer extends ContactBased implements HasTenantId { +@EqualsAndHashCode(callSuper = true) +public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; @@ -36,6 +40,9 @@ public class Customer extends ContactBased implements HasTenantId { @ApiModelProperty(position = 5, required = true, value = "JSON object with Tenant Id") private TenantId tenantId; + @Getter @Setter + private CustomerId externalId; + public Customer() { super(); } @@ -48,6 +55,7 @@ public class Customer extends ContactBased implements HasTenantId { super(customer); this.tenantId = customer.getTenantId(); this.title = customer.getTitle(); + this.externalId = customer.getExternalId(); } public TenantId getTenantId() { @@ -161,37 +169,6 @@ public class Customer extends ContactBased implements HasTenantId { return getTitle(); } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((tenantId == null) ? 0 : tenantId.hashCode()); - result = prime * result + ((title == null) ? 0 : title.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Customer other = (Customer) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; - } - @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 450c88cbb5..049c830745 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -15,16 +15,33 @@ */ package org.thingsboard.server.common.data; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; -public class Dashboard extends DashboardInfo { +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.StreamSupport; + +@EqualsAndHashCode(callSuper = true) +public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; - + private transient JsonNode configuration; - + + @Getter + @Setter + private DashboardId externalId; + public Dashboard() { super(); } @@ -40,6 +57,7 @@ public class Dashboard extends DashboardInfo { public Dashboard(Dashboard dashboard) { super(dashboard); this.configuration = dashboard.getConfiguration(); + this.externalId = dashboard.getExternalId(); } @ApiModelProperty(position = 9, value = "JSON object with main configuration of the dashboard: layouts, widgets, aliases, etc. " + @@ -54,29 +72,32 @@ public class Dashboard extends DashboardInfo { this.configuration = configuration; } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((configuration == null) ? 0 : configuration.hashCode()); - return result; + @JsonIgnore + public List getEntityAliasesConfig() { + return getChildObjects("entityAliases"); } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Dashboard other = (Dashboard) obj; - if (configuration == null) { - if (other.configuration != null) - return false; - } else if (!configuration.equals(other.configuration)) - return false; - return true; + @JsonIgnore + public List getWidgetsConfig() { + return getChildObjects("widgets"); + } + + @JsonIgnore + private List getChildObjects(String propertyName) { + return Optional.ofNullable(configuration) + .map(config -> config.get(propertyName)) + .filter(node -> !node.isEmpty()) + .map(node -> (ObjectNode) node) + .map(object -> { + List widgets = new ArrayList<>(object.size()); + object.forEach(child -> { + if (child.isObject()) { + widgets.add((ObjectNode) child); + } + }); + return widgets; + }) + .orElse(Collections.emptyList()); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java index e603f68c77..0a6d2260d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; @@ -26,6 +27,7 @@ import org.thingsboard.server.common.data.validation.NoXss; import javax.validation.Valid; import java.util.HashSet; +import java.util.Objects; import java.util.Set; @ApiModel @@ -63,7 +65,7 @@ public class DashboardInfo extends SearchTextBased implements HasNa @ApiModelProperty(position = 1, value = "JSON object with the dashboard Id. " + "Specify existing dashboard Id to update the dashboard. " + "Referencing non-existing dashboard id will cause error. " + - "Omit this field to create new dashboard." ) + "Omit this field to create new dashboard.") @Override public DashboardId getId() { return super.getId(); @@ -133,7 +135,6 @@ public class DashboardInfo extends SearchTextBased implements HasNa return this.assignedCustomers != null && this.assignedCustomers.contains(new ShortCustomerInfo(customerId, null, false)); } - public ShortCustomerInfo getAssignedCustomerInfo(CustomerId customerId) { if (this.assignedCustomers != null) { for (ShortCustomerInfo customerInfo : this.assignedCustomers) { @@ -201,25 +202,17 @@ public class DashboardInfo extends SearchTextBased implements HasNa } @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - DashboardInfo other = (DashboardInfo) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + DashboardInfo that = (DashboardInfo) o; + return mobileHide == that.mobileHide + && Objects.equals(tenantId, that.tenantId) + && Objects.equals(title, that.title) + && Objects.equals(image, that.image) + && Objects.equals(assignedCustomers, that.assignedCustomers) + && Objects.equals(mobileOrder, that.mobileOrder); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 4d7bf5e822..678c68c115 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.id.CustomerId; @@ -38,7 +40,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) @Slf4j -public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage { +public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; @@ -61,6 +63,9 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen private OtaPackageId firmwareId; private OtaPackageId softwareId; + @Getter @Setter + private DeviceId externalId; + public Device() { super(); } @@ -80,6 +85,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setDeviceData(device.getDeviceData()); this.firmwareId = device.getFirmwareId(); this.softwareId = device.getSoftwareId(); + this.externalId = device.getExternalId(); } public Device updateDevice(Device device) { @@ -93,6 +99,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setFirmwareId(device.getFirmwareId()); this.setSoftwareId(device.getSoftwareId()); Optional.ofNullable(device.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.setExternalId(device.getExternalId()); return this; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index f432862499..e087ba0b7c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -45,7 +45,7 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn @ToString(exclude = {"image", "profileDataBytes"}) @EqualsAndHashCode(callSuper = true) @Slf4j -public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage { +public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 6998485460273302018L; @@ -94,6 +94,8 @@ public class DeviceProfile extends SearchTextBased implements H @ApiModelProperty(position = 10, value = "Reference to the software OTA package. If present, the specified package will be used as default device software. ") private OtaPackageId softwareId; + private DeviceProfileId externalId; + public DeviceProfile() { super(); } @@ -116,6 +118,7 @@ public class DeviceProfile extends SearchTextBased implements H this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); this.firmwareId = deviceProfile.getFirmwareId(); this.softwareId = deviceProfile.getSoftwareId(); + this.externalId = deviceProfile.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the device profile Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java index 1f238fd672..a7e21b8a25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @AllArgsConstructor @EqualsAndHashCode(callSuper = true) public class EntityView extends SearchTextBasedWithAdditionalInfo - implements HasName, HasTenantId, HasCustomerId { + implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 5582010124562018986L; @@ -59,6 +59,8 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo @ApiModelProperty(position = 10, value = "Represents the end time of the interval that is used to limit access to target device telemetry. Customer will not be able to see entity telemetry that is outside the specified interval;") private long endTimeMs; + private EntityViewId externalId; + public EntityView() { super(); } @@ -77,6 +79,7 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo this.keys = entityView.getKeys(); this.startTimeMs = entityView.getStartTimeMs(); this.endTimeMs = entityView.getEndTimeMs(); + this.externalId = entityView.getExternalId(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java new file mode 100644 index 0000000000..64834c8b15 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ExportableEntity extends HasId, HasName { + + void setId(I id); + + I getExternalId(); + void setExternalId(I externalId); + + long getCreatedTime(); + void setCreatedTime(long createdTime); + + void setTenantId(TenantId tenantId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 17f3713b74..ff3d3c0fd4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -19,6 +19,10 @@ import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { + public static final String EMPTY = ""; + + public static final int INDEX_NOT_FOUND = -1; + public static boolean isEmpty(String source) { return source == null || source.isEmpty(); } @@ -35,6 +39,44 @@ public class StringUtils { return source != null && !source.isEmpty() && !source.trim().isEmpty(); } + public static String removeStart(final String str, final String remove) { + if (isEmpty(str) || isEmpty(remove)) { + return str; + } + if (str.startsWith(remove)){ + return str.substring(remove.length()); + } + return str; + } + + public static String substringBefore(final String str, final String separator) { + if (isEmpty(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringBetween(final String str, final String open, final String close) { + if (str == null || open == null || close == null) { + return null; + } + final int start = str.indexOf(open); + if (start != INDEX_NOT_FOUND) { + final int end = str.indexOf(close, start + open.length()); + if (end != INDEX_NOT_FOUND) { + return str.substring(start + open.length(), end); + } + } + return null; + } + public static String obfuscate(String input, int seenMargin, char obfuscationChar, int startIndexInclusive, int endIndexExclusive) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index db68df4901..da6d9615a1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -19,6 +19,9 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -33,7 +36,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) -public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId { +public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; @@ -49,6 +52,9 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements @Length(fieldName = "label") private String label; + @Getter @Setter + private AssetId externalId; + public Asset() { super(); } @@ -64,6 +70,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.name = asset.getName(); this.type = asset.getType(); this.label = asset.getLabel(); + this.externalId = asset.getExternalId(); } public void update(Asset asset) { @@ -73,6 +80,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.type = asset.getType(); this.label = asset.getLabel(); Optional.ofNullable(asset.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.externalId = asset.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the asset Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java index cfe6918a7d..4682c8f95b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; import java.io.Serializable; import java.util.UUID; @@ -33,6 +34,7 @@ public abstract class IdBased implements HasId { this.id = id; } + @JsonSetter public void setId(I id) { this.id = id; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 17d7ffef0b..eca9e331cd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; @@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Data @EqualsAndHashCode(callSuper = true) @Slf4j -public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { +public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, ExportableEntity { private static final long serialVersionUID = -5656679015121935465L; @@ -56,6 +57,8 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im @ApiModelProperty(position = 9, value = "Reserved for future usage. The actual list of rule nodes and their relations is stored in the database separately.") private transient JsonNode configuration; + private RuleChainId externalId; + @JsonIgnore private byte[] configurationBytes; @@ -75,6 +78,7 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im this.firstRuleNodeId = ruleChain.getFirstRuleNodeId(); this.root = ruleChain.isRoot(); this.setConfiguration(ruleChain.getConfiguration()); + this.setExternalId(ruleChain.getExternalId()); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java index fa2760dbc3..d9dce57f46 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java @@ -53,6 +53,8 @@ public class RuleNode extends SearchTextBasedWithAdditionalInfo impl @JsonIgnore private byte[] configurationBytes; + private RuleNodeId externalId; + public RuleNode() { super(); } @@ -68,6 +70,7 @@ public class RuleNode extends SearchTextBasedWithAdditionalInfo impl this.name = ruleNode.getName(); this.debugMode = ruleNode.isDebugMode(); this.setConfiguration(ruleNode.getConfiguration()); + this.externalId = ruleNode.getExternalId(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java new file mode 100644 index 0000000000..0e2929bef8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync; + +import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +@JacksonAnnotationsInside +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) +@JsonSubTypes({ + @Type(name = "DEVICE", value = Device.class), + @Type(name = "RULE_CHAIN", value = RuleChain.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), + @Type(name = "ASSET", value = Asset.class), + @Type(name = "DASHBOARD", value = Dashboard.class), + @Type(name = "CUSTOMER", value = Customer.class), + @Type(name = "ENTITY_VIEW", value = EntityView.class), + @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundle.class) +}) +public @interface JsonTbEntity { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java new file mode 100644 index 0000000000..1cb2ac8c74 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync; + +import org.thingsboard.server.common.data.exception.ThingsboardException; + +public interface ThrowingRunnable { + + void run() throws ThingsboardException; + + default ThrowingRunnable andThen(ThrowingRunnable after) { + return () -> { + this.run(); + after.run(); + }; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java new file mode 100644 index 0000000000..dcf69c0877 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; + +@Data +public class AttributeExportData { + private String key; + private Long lastUpdateTs; + + private Boolean booleanValue; + private String strValue; + private Long longValue; + private Double doubleValue; + private String jsonValue; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java new file mode 100644 index 0000000000..27d16899d1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class DeviceExportData extends EntityExportData { + + @JsonProperty(index = 3) + @JsonIgnoreProperties({"id", "deviceId", "createdTime"}) + private DeviceCredentials credentials; + + @JsonIgnore + @Override + public boolean hasCredentials() { + return credentials != null; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java new file mode 100644 index 0000000000..afe6c925c7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.JsonTbEntity; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) +@JsonSubTypes({ + @Type(name = "DEVICE", value = DeviceExportData.class), + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), + @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundleExportData.class) +}) +@JsonInclude(JsonInclude.Include.NON_NULL) +@Data +public class EntityExportData> { + + public static final Comparator relationsComparator = Comparator + .comparing(EntityRelation::getFrom, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTo, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTypeGroup) + .thenComparing(EntityRelation::getType); + + public static final Comparator attrComparator = Comparator + .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + + @JsonProperty(index = 2) + @JsonIgnoreProperties({"tenantId", "createdTime"}) + @JsonTbEntity + private E entity; + @JsonProperty(index = 1) + private EntityType entityType; + + @JsonProperty(index = 100) + private List relations; + @JsonProperty(index = 101) + private Map> attributes; + + public EntityExportData sort() { + if (relations != null && !relations.isEmpty()) { + relations.sort(relationsComparator); + } + if (attributes != null && !attributes.isEmpty()) { + attributes.values().forEach(list -> list.sort(attrComparator)); + } + return this; + } + + @JsonIgnore + public EntityId getExternalId() { + return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + } + + @JsonIgnore + public boolean hasCredentials() { + return false; + } + + @JsonIgnore + public boolean hasAttributes() { + return attributes != null; + } + + @JsonIgnore + public boolean hasRelations() { + return relations != null; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java new file mode 100644 index 0000000000..1a625640e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityExportSettings { + private boolean exportRelations; + private boolean exportAttributes; + private boolean exportCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java new file mode 100644 index 0000000000..1c6f4c060a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; + +@Data +public class EntityImportResult> { + + private E savedEntity; + private E oldEntity; + private EntityType entityType; + + private ThrowingRunnable saveReferencesCallback = () -> {}; + private ThrowingRunnable sendEventsCallback = () -> {}; + + private boolean updatedAllExternalIds = true; + + private boolean created; + private boolean updated; + private boolean updatedRelatedEntities; + + public void addSaveReferencesCallback(ThrowingRunnable callback) { + this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); + } + + public void addSendEventsCallback(ThrowingRunnable callback) { + this.sendEventsCallback = this.sendEventsCallback.andThen(callback); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java new file mode 100644 index 0000000000..9b95b8eca2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityImportSettings { + private boolean findExistingByName; + private boolean updateRelations; + private boolean saveAttributes; + private boolean saveCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java new file mode 100644 index 0000000000..5c37aa35b6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class RuleChainExportData extends EntityExportData { + + @JsonProperty(index = 3) + @JsonIgnoreProperties("ruleChainId") + private RuleChainMetaData metaData; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java new file mode 100644 index 0000000000..aa7600a48d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.widget.BaseWidgetType; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.Comparator; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class WidgetsBundleExportData extends EntityExportData { + + @JsonProperty(index = 3) + private List widgets; + + @Override + public EntityExportData sort() { + super.sort(); + widgets.sort(Comparator.comparing(BaseWidgetType::getAlias)); + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java new file mode 100644 index 0000000000..2120ec4c44 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; + +import java.util.HashMap; + +public class AutoCommitSettings extends HashMap { + + private static final long serialVersionUID = -5757067601838792059L; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java new file mode 100644 index 0000000000..800583b0b8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +public class EntityDataDiff { + private EntityExportData currentVersion; + private EntityExportData otherVersion; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java new file mode 100644 index 0000000000..2529ec8af6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityDataInfo { + boolean hasRelations; + boolean hasAttributes; + boolean hasCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java new file mode 100644 index 0000000000..900a05e470 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EntityLoadError { + + private String type; + private EntityId source; + private EntityId target; + + public static EntityLoadError credentialsError(EntityId sourceId) { + return EntityLoadError.builder().type("DEVICE_CREDENTIALS_CONFLICT").source(sourceId).build(); + } + + public static EntityLoadError referenceEntityError(EntityId sourceId, EntityId targetId) { + return EntityLoadError.builder().type("MISSING_REFERENCED_ENTITY").source(sourceId).target(targetId).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java new file mode 100644 index 0000000000..84a28d0770 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityTypeLoadResult { + private EntityType entityType; + private int created; + private int updated; + private int deleted; + + public EntityTypeLoadResult(EntityType entityType) { + this.entityType = entityType; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java new file mode 100644 index 0000000000..c90336d3d7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityVersion { + private long timestamp; + private String id; + private String name; + private String author; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java new file mode 100644 index 0000000000..2104472047 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionsDiff { + private EntityId externalId; + private EntityExportData entityDataAtVersion1; + private EntityExportData entityDataAtVersion2; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java new file mode 100644 index 0000000000..3c0e5849a3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +public enum RepositoryAuthMethod { + USERNAME_PASSWORD, + PRIVATE_KEY +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java new file mode 100644 index 0000000000..50e23a1dba --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.io.Serializable; + +@Data +public class RepositorySettings implements Serializable { + private static final long serialVersionUID = -3211552851889198721L; + + private String repositoryUri; + private RepositoryAuthMethod authMethod; + private String username; + private String password; + private String privateKeyFileName; + private String privateKey; + private String privateKeyPassword; + private String defaultBranch; + + public RepositorySettings() { + } + + public RepositorySettings(RepositorySettings settings) { + this.repositoryUri = settings.getRepositoryUri(); + this.authMethod = settings.getAuthMethod(); + this.username = settings.getUsername(); + this.password = settings.getPassword(); + this.privateKeyFileName = settings.getPrivateKeyFileName(); + this.privateKey = settings.getPrivateKey(); + this.privateKeyPassword = settings.getPrivateKeyPassword(); + this.defaultBranch = settings.getDefaultBranch(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java new file mode 100644 index 0000000000..8cb09b1b30 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; + +@Data +public class VersionCreationResult { + private EntityVersion version; + private int added; + private int modified; + private int removed; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java new file mode 100644 index 0000000000..736de26079 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VersionLoadResult { + + private List result; + private EntityLoadError error; + + public static VersionLoadResult success(List result) { + return VersionLoadResult.builder().result(result).build(); + } + + public static VersionLoadResult success(EntityTypeLoadResult result) { + return VersionLoadResult.builder().result(List.of(result)).build(); + } + + public static VersionLoadResult error(EntityLoadError error) { + return VersionLoadResult.builder().error(error).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java new file mode 100644 index 0000000000..fd278cde1f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@NoArgsConstructor +public class VersionedEntityInfo { + private EntityId externalId; + // etc.. + + public VersionedEntityInfo(EntityId externalId) { + this.externalId = externalId; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java new file mode 100644 index 0000000000..084b2a1059 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class AutoVersionCreateConfig extends VersionCreateConfig { + + private static final long serialVersionUID = 8245450889383315551L; + + private String branch; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java new file mode 100644 index 0000000000..f8d7279f40 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class ComplexVersionCreateRequest extends VersionCreateRequest { + + // Default sync strategy + private SyncStrategy syncStrategy; + private Map entityTypes; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.COMPLEX; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java new file mode 100644 index 0000000000..92cab96354 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateConfig extends VersionCreateConfig { + + //optional + private SyncStrategy syncStrategy; + private List entityIds; + private boolean allEntities; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java new file mode 100644 index 0000000000..507afd116e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateRequest extends VersionCreateRequest { + + private EntityId entityId; + private VersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.SINGLE_ENTITY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java new file mode 100644 index 0000000000..baf24efeb8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +public enum SyncStrategy { + MERGE, + OVERWRITE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java new file mode 100644 index 0000000000..86154bdfa0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class VersionCreateConfig implements Serializable { + private static final long serialVersionUID = 1223723167716612772L; + + private boolean saveRelations; + private boolean saveAttributes; + private boolean saveCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java new file mode 100644 index 0000000000..9a23838921 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), + @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class) +}) +@Data +public abstract class VersionCreateRequest { + + private String versionName; + private String branch; + + public abstract VersionCreateRequestType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java new file mode 100644 index 0000000000..d57363cf7d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +public enum VersionCreateRequestType { + SINGLE_ENTITY, + COMPLEX +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java new file mode 100644 index 0000000000..ffb1f25d3c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadConfig extends VersionLoadConfig { + + private boolean removeOtherEntities; + private boolean findExistingEntityByName; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java new file mode 100644 index 0000000000..f2cb83f0d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadRequest extends VersionLoadRequest { + + private Map entityTypes; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.ENTITY_TYPE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java new file mode 100644 index 0000000000..cf31317b4a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionLoadRequest extends VersionLoadRequest { + + private EntityId externalEntityId; + + private VersionLoadConfig config; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.SINGLE_ENTITY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java new file mode 100644 index 0000000000..a27cf06538 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; + +@Data +public class VersionLoadConfig { + + private boolean loadRelations; + private boolean loadAttributes; + private boolean loadCredentials; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java new file mode 100644 index 0000000000..d9d1329c8b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; + +import static com.fasterxml.jackson.annotation.JsonSubTypes.Type; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionLoadRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionLoadRequest.class) +}) +@Data +public abstract class VersionLoadRequest { + + private String branch; + private String versionId; + + public abstract VersionLoadRequestType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java new file mode 100644 index 0000000000..190e6871d5 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +public enum VersionLoadRequestType { + SINGLE_ENTITY, + ENTITY_TYPE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 1cd49a6729..e7d6383540 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,6 +44,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String tenantEntityExportRateLimit; + private String tenantEntityImportRateLimit; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java index 1515aceb62..4f3de71512 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.common.data.widget; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBased; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +30,8 @@ import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class WidgetsBundle extends SearchTextBased implements HasTenantId { +@EqualsAndHashCode(callSuper = true) +public class WidgetsBundle extends SearchTextBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -7627368878362410489L; @@ -63,6 +67,10 @@ public class WidgetsBundle extends SearchTextBased implements H @ApiModelProperty(position = 7, value = "Description", readOnly = true) private String description; + @Getter + @Setter + private WidgetsBundleId externalId; + public WidgetsBundle() { super(); } @@ -78,6 +86,7 @@ public class WidgetsBundle extends SearchTextBased implements H this.title = widgetsBundle.getTitle(); this.image = widgetsBundle.getImage(); this.description = widgetsBundle.getDescription(); + this.externalId = widgetsBundle.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the Widget Bundle Id. " + @@ -100,31 +109,10 @@ public class WidgetsBundle extends SearchTextBased implements H return getTitle(); } + @JsonIgnore @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (tenantId != null ? tenantId.hashCode() : 0); - result = 31 * result + (alias != null ? alias.hashCode() : 0); - result = 31 * result + (title != null ? title.hashCode() : 0); - result = 31 * result + (image != null ? image.hashCode() : 0); - result = 31 * result + (description != null ? description.hashCode() : 0); - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - WidgetsBundle that = (WidgetsBundle) o; - - if (tenantId != null ? !tenantId.equals(that.tenantId) : that.tenantId != null) return false; - if (alias != null ? !alias.equals(that.alias) : that.alias != null) return false; - if (title != null ? !title.equals(that.title) : that.title != null) return false; - if (image != null ? !image.equals(that.image) : that.image != null) return false; - if (description != null ? !description.equals(that.description) : that.description != null) return false; - return true; + public String getName() { + return title; } @Override @@ -133,7 +121,6 @@ public class WidgetsBundle extends SearchTextBased implements H sb.append("tenantId=").append(tenantId); sb.append(", alias='").append(alias).append('\''); sb.append(", title='").append(title).append('\''); - sb.append(", image='").append(image).append('\''); sb.append(", description='").append(description).append('\''); sb.append('}'); return sb.toString(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 1c05ac7f66..7f276e7e3f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR, TB_VC_EXECUTOR; public static ServiceType of(String serviceType) { return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); diff --git a/common/pom.xml b/common/pom.xml index 236fb50a19..62bb46e81c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,7 @@ cache coap-server edge-api + version-control diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e2197d566f..a625ec6e93 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,6 +120,10 @@ com.google.protobuf protobuf-java-util + + de.ruedigermoeller + fst + org.apache.curator curator-recipes diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java index 9fa91a4e40..b87049b634 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java @@ -37,7 +37,8 @@ public class TbServiceBusQueueConfigs { private String notificationsProperties; @Value("${queue.service-bus.queue-properties.js-executor}") private String jsExecutorProperties; - + @Value("${queue.service-bus.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreConfigs; @Getter @@ -48,6 +49,8 @@ public class TbServiceBusQueueConfigs { private Map notificationsConfigs; @Getter private Map jsExecutorConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -56,6 +59,7 @@ public class TbServiceBusQueueConfigs { transportApiConfigs = getConfigs(transportApiProperties); notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e38677b2ce..9fbe9b1104 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate new ArrayList<>()).add(instance); } }); - } else if (ServiceType.TB_CORE.equals(serviceType)) { + } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java index 31bad5477e..7b5c142c23 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java @@ -40,7 +40,7 @@ public abstract class TbApplicationEventListener i } finally { seqNumberLock.unlock(); } - if (validUpdate) { + if (validUpdate && filterTbApplicationEvent(event)) { onTbApplicationEvent(event); } else { log.info("Application event ignored due to invalid sequence number ({} > {}). Event: {}", lastProcessedSequenceNumber, event.getSequenceNumber(), event); @@ -49,5 +49,8 @@ public abstract class TbApplicationEventListener i protected abstract void onTbApplicationEvent(T event); + protected boolean filterTbApplicationEvent(T event) { + return true; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 89eda16cad..5976a1372b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,7 +33,6 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 4c9194b675..75d0bd9b2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -40,6 +40,9 @@ public class TbKafkaTopicConfigs { private String jsExecutorProperties; @Value("${queue.kafka.topic-properties.ota-updates:}") private String fwUpdatesProperties; + @Value("${queue.kafka.topic-properties.version-control:}") + private String vcProperties; + @Getter private Map coreConfigs; @@ -53,6 +56,8 @@ public class TbKafkaTopicConfigs { private Map jsExecutorConfigs; @Getter private Map fwUpdatesConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -62,6 +67,7 @@ public class TbKafkaTopicConfigs { notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); fwUpdatesConfigs = getConfigs(fwUpdatesProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 660173c71d..2ca5090aed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -46,6 +47,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -57,7 +59,7 @@ import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; @@ -66,6 +68,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; @@ -73,6 +76,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public AwsSqsMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -80,6 +84,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, + TbQueueVersionControlSettings vcSettings, TbAwsSqsQueueAttributes sqsQueueAttributes, TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.notificationsTopicService = notificationsTopicService; @@ -89,6 +94,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; + this.vcSettings = vcSettings; this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); @@ -96,6 +102,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); } @Override @@ -123,6 +130,13 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, configuration.getTopic(), @@ -205,6 +219,11 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -222,5 +241,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 81736c2b55..89b45e2822 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..d76b3f595d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-vc-executor'") +public class AwsSqsTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public AwsSqsTbVersionControlQueueFactory(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes + ) { + this.sqsSettings = sqsSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fe87fc2886..815e0efdf8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -37,28 +37,32 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueVersionControlSettings vcSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, InMemoryStorage storage) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; + this.vcSettings = vcSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -91,6 +95,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, vcSettings.getTopic()); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new InMemoryTbQueueConsumer<>(storage, configuration.getTopic()); @@ -146,6 +155,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, vcSettings.getTopic()); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 104c531e12..b24ae700bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,6 +52,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicLong; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbKafkaSettings kafkaSettings; @@ -68,6 +70,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -76,6 +79,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -85,6 +90,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.notificationsTopicService = notificationsTopicService; @@ -95,6 +101,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -103,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -155,6 +163,19 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("monolith-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { String queueName = configuration.getName(); @@ -311,6 +332,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -331,5 +362,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 58dec0792a..f476a068d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -50,6 +51,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -65,6 +67,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; @@ -74,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; public KafkaTbCoreQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -82,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { @@ -92,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; @@ -101,6 +107,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -282,6 +289,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -302,5 +319,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..598f86d0f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-vc-executor'") +public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbKafkaConsumerStatsService consumerStatsService, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + this.consumerStatsService = consumerStatsService; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("tb-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index aceb9d4f0d..d439bb0ed7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,13 +52,14 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -67,12 +69,14 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final NotificationsTopicService notificationsTopicService; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -82,7 +86,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng NotificationsTopicService notificationsTopicService, TbServiceInfoProvider serviceInfoProvider, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -90,12 +95,15 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportNotificationSettings = transportNotificationSettings; this.notificationsTopicService = notificationsTopicService; this.serviceInfoProvider = serviceInfoProvider; + this.vcSettings = vcSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + this.vcAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getVcSettings()); + this.jsInvokeSettings = jsInvokeSettings; } @@ -125,6 +133,13 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, configuration.getTopic(), @@ -207,6 +222,11 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -224,5 +244,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 18a6668581..6cf9aa9a45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..45e71b2299 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-vc-executor'") +public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public PubSubTbVersionControlQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings + ) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + this.vcAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getVcSettings()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 2a724289bb..3223e46cc5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,13 +52,14 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") -public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; @@ -67,12 +69,14 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public RabbitMqMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -81,7 +85,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, TbRabbitMqQueueArguments queueArguments, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -90,12 +95,14 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + this.vcAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getVcArgs()); } @Override @@ -123,6 +130,13 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(vcAdmin, rabbitMqSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, configuration.getTopic(), @@ -205,6 +219,11 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -222,5 +241,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e728be6085..1dca3ba551 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -169,6 +170,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..efc433c86e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-vc-executor'") +public class RabbitMqTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public RabbitMqTbVersionControlQueueFactory(TbRabbitMqSettings rabbitMqSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbRabbitMqQueueArguments queueArguments + ) { + this.rabbitMqSettings = rabbitMqSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getCoreArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getNotificationsArgs()); + this.vcAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getVcArgs()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(vcAdmin, rabbitMqSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 88bb0a4045..a740a6d1a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -50,13 +51,14 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") -public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; @@ -66,12 +68,14 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public ServiceBusMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -80,6 +84,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; @@ -89,12 +94,14 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); + this.vcAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getVcConfigs()); } @Override @@ -122,6 +129,13 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(vcAdmin, serviceBusSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, configuration.getTopic(), @@ -204,6 +218,11 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -221,5 +240,8 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e1eb41b2b2..5b471b2830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..75e788af16 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-vc-executor'") +public class ServiceBusTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public ServiceBusTbVersionControlQueueFactory(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbServiceBusQueueConfigs serviceBusQueueConfigs + ) { + this.serviceBusSettings = serviceBusSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); + this.vcAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getVcConfigs()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(vcAdmin, serviceBusSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 4debe1e280..aadefa0292 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateSer import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -122,4 +123,11 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueProducer> createTransportApiResponseProducer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + /** + * Used to push messages to instances of TB Version Control Service + * + * @return + */ + TbQueueProducer> createVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index ef0b0c38e9..ed7de35274 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -22,6 +22,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -39,6 +40,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toVersionControl; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -52,6 +54,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); } @Override @@ -83,4 +86,9 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + return toVersionControl; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 19ebb4666e..19046c5a2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -70,4 +71,11 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbUsageStatsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 26f9df069e..c21ae99b8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -83,4 +84,9 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 3132ed684f..f9cc139559 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -73,6 +74,11 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + @Override public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java new file mode 100644 index 0000000000..c8ffc1da0d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-vc-executor'") +public class TbVersionControlProducerProvider implements TbQueueProducerProvider { + + private final TbVersionControlQueueFactory tbQueueProvider; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; + + public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java new file mode 100644 index 0000000000..70b16afc68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages from TB Core Service + * + * @return + */ + TbQueueConsumer> createToVersionControlMsgConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java index e849d5b4f3..a4819fdad4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -37,6 +37,8 @@ public class TbPubSubSubscriptionSettings { private String notificationsProperties; @Value("${queue.pubsub.queue-properties.js-executor}") private String jsExecutorProperties; + @Value("${queue.pubsub.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreSettings; @@ -48,6 +50,8 @@ public class TbPubSubSubscriptionSettings { private Map notificationsSettings; @Getter private Map jsExecutorSettings; + @Getter + private Map vcSettings; @PostConstruct private void init() { @@ -56,6 +60,7 @@ public class TbPubSubSubscriptionSettings { transportApiSettings = getSettings(transportApiProperties); notificationsSettings = getSettings(notificationsProperties); jsExecutorSettings = getSettings(jsExecutorProperties); + vcSettings = getSettings(vcProperties); } private Map getSettings(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java index d2523f3b8f..25c78a96c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -38,6 +38,8 @@ public class TbRabbitMqQueueArguments { private String notificationsProperties; @Value("${queue.rabbitmq.queue-properties.js-executor}") private String jsExecutorProperties; + @Value("${queue.rabbitmq.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreArgs; @@ -49,6 +51,8 @@ public class TbRabbitMqQueueArguments { private Map notificationsArgs; @Getter private Map jsExecutorArgs; + @Getter + private Map vcArgs; @PostConstruct private void init() { @@ -57,6 +61,7 @@ public class TbRabbitMqQueueArguments { transportApiArgs = getArgs(transportApiProperties); notificationsArgs = getArgs(notificationsProperties); jsExecutorArgs = getArgs(jsExecutorProperties); + vcArgs = getArgs(vcProperties); } private Map getArgs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 33e6552eb9..f64143b14c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueCoreSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index b930d04312..f4d62bad1d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRemoteJsInvokeSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index eee70e9e22..57131f6b62 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 0ad9414bd8..a4dfe86bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportApiSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index d4c0064692..e2a10b31b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportNotificationSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java new file mode 100644 index 0000000000..3a7a5602d5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueVersionControlSettings { + + @Value("${queue.vc.topic:tb_version_control}") + private String topic; + + @Value("${queue.vc.usage-stats-topic:tb_usage_stats}") + private String usageStatsTopic; + + @Value("${queue.vc.partitions:10}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 0cf02348f3..58704f62a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -38,6 +38,8 @@ public class TbAwsSqsQueueAttributes { private String notificationsProperties; @Value("${queue.aws-sqs.queue-properties.js-executor}") private String jsExecutorProperties; + @Value("${queue.aws-sqs.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreAttributes; @@ -49,6 +51,8 @@ public class TbAwsSqsQueueAttributes { private Map notificationsAttributes; @Getter private Map jsExecutorAttributes; + @Getter + private Map vcAttributes; private final Map defaultAttributes = new HashMap<>(); @@ -61,6 +65,7 @@ public class TbAwsSqsQueueAttributes { transportApiAttributes = getConfigs(transportApiProperties); notificationsAttributes = getConfigs(notificationsProperties); jsExecutorAttributes = getConfigs(jsExecutorProperties); + vcAttributes = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java index ed0572e8e5..8ebcc39cab 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import java.util.Optional; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index 1c5eec383c..f1c55973db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -13,11 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import java.util.Optional; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java new file mode 100644 index 0000000000..e72fe60b6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-vc-executor'") +public @interface TbVersionControlComponent { +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index c75de25267..a10c3e4969 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -24,14 +24,11 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @TbSnmpTransportComponent @Service diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 8b3f9feb22..51373e2a4a 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -64,10 +64,6 @@ com.google.code.gson gson - - de.ruedigermoeller - fst - org.slf4j slf4j-api diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index b79306199f..2b0f73343d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java index 666941b143..b4187432e8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java @@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.TransportResourceCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 9f6ddc87a4..68f8db272b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -69,7 +69,7 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; @@ -901,7 +901,7 @@ public class DefaultTransportService implements TransportService { if (EntityType.DEVICE_PROFILE.equals(entityType)) { DeviceProfile deviceProfile = deviceProfileCache.put(msg.getData()); if (deviceProfile != null) { - log.info("On device profile update: {}", deviceProfile); + log.debug("On device profile update: {}", deviceProfile); onProfileUpdate(deviceProfile); } } else if (EntityType.TENANT_PROFILE.equals(entityType)) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 4cfefb7a6f..bb7f8f3f4d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index f9df92966b..9c4b68d3c0 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -18,13 +18,20 @@ package org.thingsboard.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.UUID; +import java.util.function.UnaryOperator; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -32,6 +39,11 @@ import java.util.Set; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final ObjectMapper PRETTY_SORTED_JSON_MAPPER = JsonMapper.builder() + .enable(SerializationFeature.INDENT_OUTPUT) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) + .build(); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -96,6 +108,14 @@ public class JacksonUtil { } } + public static String toPrettyString(Object o) { + try { + return PRETTY_SORTED_JSON_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public static JsonNode toJsonNode(String value) { if (value == null || value.isEmpty()) { return null; @@ -137,4 +157,59 @@ public class JacksonUtil { + value + " cannot be transformed to a String", e); } } + + + public static JsonNode getSafely(JsonNode node, String... path) { + if (node == null) { + return null; + } + for (String p : path) { + if (!node.has(p)) { + return null; + } else { + node = node.get(p); + } + } + return node; + } + + public static void replaceUuidsRecursively(JsonNode node, Set skipFieldsSet, UnaryOperator replacer) { + if (node == null) { + return; + } + if (node.isObject()) { + ObjectNode objectNode = (ObjectNode) node; + List fieldNames = new ArrayList<>(objectNode.size()); + objectNode.fieldNames().forEachRemaining(fieldNames::add); + for (String fieldName : fieldNames) { + if (skipFieldsSet.contains(fieldName)) { + continue; + } + var child = objectNode.get(fieldName); + if (child.isObject() || child.isArray()) { + replaceUuidsRecursively(child, skipFieldsSet, replacer); + } else if (child.isTextual()) { + String text = child.asText(); + String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); + if (!text.equals(newText)) { + objectNode.put(fieldName, newText); + } + } + } + } else if (node.isArray()) { + ArrayNode array = (ArrayNode) node; + for (int i = 0; i < array.size(); i++) { + JsonNode arrayElement = array.get(i); + if (arrayElement.isObject() || arrayElement.isArray()) { + replaceUuidsRecursively(arrayElement, skipFieldsSet, replacer); + } else if (arrayElement.isTextual()) { + String text = arrayElement.asText(); + String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); + if (!text.equals(newText)) { + array.set(i, newText); + } + } + } + } + } } diff --git a/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java b/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java new file mode 100644 index 0000000000..af968a6bff --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.function.UnaryOperator; +import java.util.regex.Pattern; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class RegexUtils { + + public static final Pattern UUID_PATTERN = Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + + + public static String replace(String s, Pattern pattern, UnaryOperator replacer) { + return pattern.matcher(s).replaceAll(matchResult -> { + return replacer.apply(matchResult.group()); + }); + } + +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index 13a0d466ac..d8b4cbf720 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -28,12 +28,23 @@ import org.springframework.util.StopWatch; * */ public class TbStopWatch extends StopWatch { - public static TbStopWatch startNew(){ + public static TbStopWatch create(){ TbStopWatch stopWatch = new TbStopWatch(); stopWatch.start(); return stopWatch; } + public static TbStopWatch create(String taskName){ + TbStopWatch stopWatch = new TbStopWatch(); + stopWatch.start(taskName); + return stopWatch; + } + + public void startNew(String taskName){ + stop(); + start(taskName); + } + public long stopAndGetTotalTimeMillis(){ stop(); return getTotalTimeMillis(); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml new file mode 100644 index 0000000000..998ae0879b --- /dev/null +++ b/common/version-control/pom.xml @@ -0,0 +1,125 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + common + + org.thingsboard.common + version-control + jar + + Thingsboard Server Version Control API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.thingsboard.common + data + + + org.thingsboard.common + queue + + + org.springframework + spring-core + + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.eclipse.jgit + org.eclipse.jgit + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java new file mode 100644 index 0000000000..0823dd06e6 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface ClusterVersionControlService extends ApplicationListener { +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java new file mode 100644 index 0000000000..b28618372e --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -0,0 +1,542 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.AddMsg; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityVersionProto; +import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.VersionedEntityInfoProto; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbVersionControlComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.thingsboard.server.service.sync.vc.DefaultGitRepositoryService.fromRelativePath; + +@Slf4j +@TbVersionControlComponent +@Service +@RequiredArgsConstructor +public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { + + private final PartitionService partitionService; + private final TbQueueProducerProvider producerProvider; + private final TbVersionControlQueueFactory queueFactory; + private final DataDecodingEncodingService encodingService; + private final GitRepositoryService vcService; + private final NotificationsTopicService notificationsTopicService; + + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + private volatile ExecutorService consumerExecutor; + private volatile TbQueueConsumer> consumer; + private volatile TbQueueProducer> producer; + private volatile boolean stopped = false; + + @Value("${queue.vc.poll-interval:25}") + private long pollDuration; + @Value("${queue.vc.pack-processing-timeout:60000}") + private long packProcessingTimeout; + @Value("${vc.git.io_pool_size:3}") + private int ioPoolSize; + + //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. + private final List ioThreads = new ArrayList<>(); + + + @PostConstruct + public void init() { + consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); + var threadFactory = ThingsBoardThreadFactory.forName("vc-io-thread"); + for (int i = 0; i < ioPoolSize; i++) { + ioThreads.add(MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(threadFactory))); + } + producer = producerProvider.getTbCoreNotificationsMsgProducer(); + consumer = queueFactory.createToVersionControlMsgConsumer(); + } + + @PreDestroy + public void stop() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + ioThreads.forEach(ExecutorService::shutdownNow); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + for (TenantId tenantId : vcService.getActiveRepositoryTenants()) { + if (!partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId).isMyPartition()) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + pendingCommitMap.remove(tenantId); + vcService.clearRepository(tenantId); + } catch (Exception e) { + log.warn("[{}] Failed to cleanup the tenant repository", tenantId, e); + } finally { + lock.unlock(); + } + } + } + consumer.subscribe(event.getPartitions()); + } + + @Override + protected boolean filterTbApplicationEvent(PartitionChangeEvent event) { + return ServiceType.TB_VC_EXECUTOR.equals(event.getServiceType()); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent event) { + consumerExecutor.execute(() -> consumerLoop(consumer)); + } + + void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { + List> futures = new ArrayList<>(); + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msgWrapper : msgs) { + ToVersionControlServiceMsg msg = msgWrapper.getValue(); + var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); + long startTs = System.currentTimeMillis(); + log.trace("[{}][{}] RECEIVED task: {}", ctx.getTenantId(), ctx.getRequestId(), msg); + int threadIdx = Math.abs(ctx.getTenantId().hashCode() % ioPoolSize); + ListenableFuture future = ioThreads.get(threadIdx).submit(() -> processMessage(ctx, msg)); + logTaskExecution(ctx, future, startTs); + futures.add(future); + } + try { + Futures.allAsList(futures).get(packProcessingTimeout, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.info("Timeout for processing the version control tasks.", e); + } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain version control requests from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new version control messages", e2); + } + } + } + } + log.info("TB Version Control request consumer stopped."); + } + + private Void processMessage(VersionControlRequestCtx ctx, ToVersionControlServiceMsg msg) { + var lock = getRepoLock(ctx.getTenantId()); + lock.lock(); + try { + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(ctx); + } else { + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } else { + var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); + var newSettings = ctx.getSettings(); + if (!newSettings.equals(currentSettings)) { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } + if (msg.hasCommitRequest()) { + handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } else if (msg.hasVersionsDiffRequest()) { + handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); + } else if (msg.hasContentsDiffRequest()) { + handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); + } + } + } + } catch (Exception e) { + reply(ctx, Optional.of(e)); + } finally { + lock.unlock(); + } + return null; + } + + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { + var entityType = EntityType.valueOf(request.getEntityType()); + String path = getRelativePath(entityType, null); + var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) + .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); + var response = EntitiesContentResponseMsg.newBuilder(); + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + response.addData(data); + } + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + } + + private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { + String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + } + + private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { + String path; + if (StringUtils.isNotEmpty(request.getEntityType())) { + var entityType = EntityType.valueOf(request.getEntityType()); + if (request.getEntityIdLSB() != 0 || request.getEntityIdMSB() != 0) { + path = getRelativePath(entityType, new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + } else { + path = getRelativePath(entityType, null); + } + } else { + path = null; + } + SortOrder sortOrder = null; + if (StringUtils.isNotEmpty(request.getSortProperty())) { + var direction = SortOrder.Direction.DESC; + if (StringUtils.isNotEmpty(request.getSortDirection())) { + direction = SortOrder.Direction.valueOf(request.getSortDirection()); + } + sortOrder = new SortOrder(request.getSortProperty(), direction); + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, + new PageLink(request.getPageSize(), request.getPage(), request.getTextSearch(), sortOrder)); + reply(ctx, Optional.empty(), builder -> + builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() + .setTotalPages(data.getTotalPages()) + .setTotalElements(data.getTotalElements()) + .setHasNext(data.hasNext()) + .addAllVersions(data.getData().stream().map( + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).setAuthor(v.getAuthor()).build() + ).collect(Collectors.toList()))) + ); + } + + private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception { + EntityType entityType = StringUtils.isNotEmpty(request.getEntityType()) ? EntityType.valueOf(request.getEntityType()) : null; + var path = entityType != null ? getRelativePath(entityType, null) : null; + var data = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListEntitiesResponse(ListEntitiesResponseMsg.newBuilder() + .addAllEntities(data.stream().map(VersionedEntityInfo::getExternalId).map( + id -> VersionedEntityInfoProto.newBuilder() + .setEntityType(id.getEntityType().name()) + .setEntityIdMSB(id.getId().getMostSignificantBits()) + .setEntityIdLSB(id.getId().getLeastSignificantBits()).build() + ).collect(Collectors.toList())))); + } + + private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { + var branches = vcService.listBranches(ctx.getTenantId()); + reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); + } + + private void handleVersionsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.VersionsDiffRequestMsg request) throws IOException { + List diffList = vcService.getVersionsDiffList(ctx.getTenantId(), request.getPath(), request.getVersionId1(), request.getVersionId2()).stream() + .map(diff -> { + EntityId entityId = fromRelativePath(diff.getFilePath()); + return TransportProtos.EntityVersionsDiff.newBuilder() + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEntityDataAtVersion1(diff.getFileContentAtCommit1()) + .setEntityDataAtVersion2(diff.getFileContentAtCommit2()) + .setRawDiff(diff.getDiffStringValue()) + .build(); + }) + .collect(Collectors.toList()); + + reply(ctx, builder -> builder.setVersionsDiffResponse(TransportProtos.VersionsDiffResponseMsg.newBuilder() + .addAllDiff(diffList))); + } + + private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { + String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); + reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() + .setDiff(diff))); + } + + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { + var tenantId = ctx.getTenantId(); + UUID txId = UUID.fromString(request.getTxId()); + if (request.hasPrepareMsg()) { + vcService.fetch(ctx.getTenantId()); + prepareCommit(ctx, txId, request.getPrepareMsg()); + } else if (request.hasAbortMsg()) { + PendingCommit current = pendingCommitMap.get(tenantId); + if (current != null && current.getTxId().equals(txId)) { + doAbortCurrentCommit(tenantId, current); + } + } else { + PendingCommit current = pendingCommitMap.get(tenantId); + if (current != null && current.getTxId().equals(txId)) { + try { + if (request.hasAddMsg()) { + addToCommit(ctx, current, request.getAddMsg()); + } else if (request.hasDeleteMsg()) { + deleteFromCommit(ctx, current, request.getDeleteMsg()); + } else if (request.hasPushMsg()) { + var result = vcService.push(current); + pendingCommitMap.remove(ctx.getTenantId()); + reply(ctx, result); + } + } catch (Exception e) { + doAbortCurrentCommit(tenantId, current, e); + throw e; + } + } else { + log.debug("[{}] Ignore request due to stale commit: {}", txId, request); + } + } + } + + private void prepareCommit(VersionControlRequestCtx ctx, UUID txId, PrepareMsg prepareMsg) { + var tenantId = ctx.getTenantId(); + var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), + prepareMsg.getCommitMsg(), prepareMsg.getAuthorName(), prepareMsg.getAuthorEmail()); + PendingCommit old = pendingCommitMap.get(tenantId); + if (old != null) { + doAbortCurrentCommit(tenantId, old); + } + pendingCommitMap.put(tenantId, pendingCommit); + vcService.prepareCommit(pendingCommit); + } + + private void deleteFromCommit(VersionControlRequestCtx ctx, PendingCommit commit, DeleteMsg deleteMsg) throws IOException { + vcService.deleteFolderContent(commit, deleteMsg.getRelativePath()); + } + + private void addToCommit(VersionControlRequestCtx ctx, PendingCommit commit, AddMsg addMsg) throws IOException { + vcService.add(commit, addMsg.getRelativePath(), addMsg.getEntityDataJson()); + } + + private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current) { + doAbortCurrentCommit(tenantId, current, null); + } + + private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current, Exception e) { + vcService.abort(current); + pendingCommitMap.remove(tenantId); + //TODO: push notification to core using old.getNodeId() to cancel old commit processing on the caller side. + } + + private void handleClearRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.clearRepository(ctx.getTenantId()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void handleInitRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + + private void handleTestRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.testRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { + var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded()) + .setModified(result.getModified()) + .setRemoved(result.getRemoved()); + + if (result.getVersion() != null) { + responseBuilder.setTs(result.getVersion().getTimestamp()) + .setCommitId(result.getVersion().getId()) + .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()); + } + + reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(responseBuilder)); + } + + private void reply(VersionControlRequestCtx ctx, Optional e) { + reply(ctx, e, null); + } + + private void reply(VersionControlRequestCtx ctx, Function enrichFunction) { + reply(ctx, Optional.empty(), enrichFunction); + } + + private void reply(VersionControlRequestCtx ctx, Optional e, Function enrichFunction) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); + VersionControlResponseMsg.Builder builder = VersionControlResponseMsg.newBuilder() + .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) + .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); + if (e.isPresent()) { + log.debug("[{}][{}] Failed to process task", ctx.getTenantId(), ctx.getRequestId(), e.get()); + builder.setError(e.get().getMessage()); + } else { + if (enrichFunction != null) { + builder = enrichFunction.apply(builder); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + log.debug("[{}][{}] Processed task", ctx.getTenantId(), ctx.getRequestId()); + } + + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); + log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + } + + private RepositorySettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + if (settingsOpt.isPresent()) { + return settingsOpt.get(); + } else { + log.warn("Failed to parse VC settings: {}", msg.getVcSettings()); + throw new RuntimeException("Failed to parse vc settings!"); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); + } + + private void logTaskExecution(VersionControlRequestCtx ctx, ListenableFuture future, long startTs) { + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback() { + + @Override + public void onSuccess(@Nullable Object result) { + log.trace("[{}][{}] Task processing took: {}ms", ctx.getTenantId(), ctx.getRequestId(), (System.currentTimeMillis() - startTs)); + } + + @Override + public void onFailure(Throwable t) { + log.trace("[{}][{}] Task failed: ", ctx.getTenantId(), ctx.getRequestId(), t); + } + }, MoreExecutors.directExecutor()); + } + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..99dbebc3c9 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,275 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; + +import javax.annotation.PostConstruct; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${java.io.tmpdir}/repositories") + private String defaultFolder; + + @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(repositoriesFolder)) { + repositoriesFolder = defaultFolder; + } + } + + @Override + public Set getActiveRepositoryTenants() { + return new HashSet<>(repositories.keySet()); + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getBranch(); + try { + repository.fetch(); + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(branch)) { + repository.merge(branch); + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + if (result.getAdded() > 0 || result.getModified() > 0 || result.getRemoved() > 0) { + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); + repository.push(commit.getWorkingBranch(), commit.getBranch()); + result.setVersion(toVersion(gitCommit)); + } + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); + } + } + + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + log.debug("[{}] Cleanup tenant repository started.", commit.getTenantId()); + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + log.debug("[{}] Cleanup tenant repository completed.", commit.getTenantId()); + } + + @Override + public void abort(PendingCommit commit) { + cleanUp(commit); + } + + @Override + public void fetch(TenantId tenantId) throws GitAPIException { + var repository = repositories.get(tenantId); + if (repository != null) { + log.debug("[{}] Fetching tenant repository.", tenantId); + repository.fetch(); + log.debug("[{}] Fetched tenant repository.", tenantId); + } + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getDiffList(versionId1, versionId2, path); + } + + @Override + public String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getContentsDiff(content1, content2); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listRemoteBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository.test(settings, repositoryDirectory.toFile()); + } + + @Override + public void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception { + clearRepository(tenantId); + log.debug("[{}] Init tenant repository started.", tenantId); + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings, repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + log.debug("[{}] Init tenant repository completed.", tenantId); + } + + @Override + public RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception { + var gitRepository = repositories.get(tenantId); + return gitRepository != null ? gitRepository.getSettings() : null; + } + + @Override + public void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + log.debug("[{}] Clear tenant repository started.", tenantId); + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + log.debug("[{}] Clear tenant repository completed.", tenantId); + } + } + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage(), this.getAuthor(commit)); + } + + private String getAuthor(GitRepository.Commit commit) { + String author = String.format("<%s>", commit.getAuthorEmail()); + if (StringUtils.isNotBlank(commit.getAuthorName())) { + author = String.format("%s %s", commit.getAuthorName(), author); + } + return author; + } + + public static EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java new file mode 100644 index 0000000000..2b2802d003 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -0,0 +1,494 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.common.collect.Streams; +import lombok.Data; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.apache.sshd.common.util.security.SecurityUtils; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.diff.EditList; +import org.eclipse.jgit.diff.HistogramDiff; +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.diff.RawTextComparator; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.SshTransport; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.sshd.JGitKeyCache; +import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; +import org.eclipse.jgit.transport.sshd.SshdSessionFactory; +import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PublicKey; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class GitRepository { + + private final Git git; + @Getter + private final RepositorySettings settings; + private final CredentialsProvider credentialsProvider; + private final SshdSessionFactory sshSessionFactory; + + @Getter + private final String directory; + + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { + this.git = git; + this.settings = settings; + this.credentialsProvider = credentialsProvider; + this.sshSessionFactory = sshSessionFactory; + this.directory = directory; + } + + public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + CloneCommand cloneCommand = Git.cloneRepository() + .setURI(settings.getRepositoryUri()) + .setDirectory(directory) + .setNoCheckout(true); + configureTransportCommand(cloneCommand, credentialsProvider, sshSessionFactory); + Git git = cloneCommand.call(); + return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); + } + + public static GitRepository open(File directory, RepositorySettings settings) throws IOException { + Git git = Git.open(directory); + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); + } + + public static void test(RepositorySettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); + configureTransportCommand(lsRemoteCommand, credentialsProvider, sshSessionFactory); + lsRemoteCommand.call(); + } + + public void fetch() throws GitAPIException { + execute(git.fetch() + .setRemoveDeletedRefs(true)); + } + + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); + } + + public void resetAndClean() throws GitAPIException { + execute(git.reset() + .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); + } + + public void merge(String branch) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + + + public List listRemoteBranches() throws GitAPIException { + return execute(git.branchList() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() + .filter(ref -> !ref.getName().equals(Constants.HEAD)) + .map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) + .map(name -> StringUtils.removeStart(name, "origin/")) + .distinct().collect(Collectors.toList()); + } + + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); + } + + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + return new PageData<>(); + } + LogCommand command = git.log() + .add(branchId); + + if (StringUtils.isNotEmpty(pageLink.getTextSearch())) { + command.setRevFilter(new NoMergesAndCommitMessageFilter(pageLink.getTextSearch())); + } else { + command.setRevFilter(RevFilter.NO_MERGES); + } + + if (StringUtils.isNotEmpty(path)) { + command.addPath(path); + } + + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); + } + + public List listFilesAtCommit(String commitId) throws IOException { + return listFilesAtCommit(commitId, null); + } + + public List listFilesAtCommit(String commitId, String path) throws IOException { + List files = new ArrayList<>(); + RevCommit revCommit = resolveCommit(commitId); + try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { + treeWalk.reset(revCommit.getTree().getId()); + if (StringUtils.isNotEmpty(path)) { + treeWalk.setFilter(PathFilter.create(path)); + } + treeWalk.setRecursive(true); + while (treeWalk.next()) { + files.add(treeWalk.getPathString()); + } + } + return files; + } + + + public String getFileContentAtCommit(String file, String commitId) throws IOException { + RevCommit revCommit = resolveCommit(commitId); + try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { + if (treeWalk == null) { + throw new IllegalArgumentException("File not found"); + } + ObjectId blobId = treeWalk.getObjectId(0); + try (ObjectReader objectReader = git.getRepository().newObjectReader()) { + ObjectLoader objectLoader = objectReader.open(blobId); + byte[] bytes = objectLoader.getBytes(); + return new String(bytes, StandardCharsets.UTF_8); + } + } + } + + + public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { + execute(git.checkout() + .setOrphan(true) + .setForced(true) + .setName(name)); + } + + public void add(String filesPattern) throws GitAPIException { + execute(git.add().setUpdate(true).addFilepattern(filesPattern)); + execute(git.add().addFilepattern(filesPattern)); + } + + public Status status() throws GitAPIException { + org.eclipse.jgit.api.Status status = execute(git.status()); + Set modified = new HashSet<>(); + modified.addAll(status.getModified()); + modified.addAll(status.getChanged()); + return new Status(status.getAdded(), modified, status.getRemoved()); + } + + public Commit commit(String message, String authorName, String authorEmail) throws GitAPIException { + RevCommit revCommit = execute(git.commit() + .setAuthor(authorName, authorEmail) + .setMessage(message)); + return toCommit(revCommit); + } + + + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); + } + + public String getContentsDiff(String content1, String content2) throws IOException { + RawText rawContent1 = new RawText(content1.getBytes()); + RawText rawContent2 = new RawText(content2.getBytes()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + + EditList edits = new EditList(); + edits.addAll(new HistogramDiff().diff(RawTextComparator.DEFAULT, rawContent1, rawContent2)); + diffFormatter.format(edits, rawContent1, rawContent2); + return out.toString(); + } + + public List getDiffList(String commit1, String commit2, String path) throws IOException { + ObjectReader reader = git.getRepository().newObjectReader(); + + CanonicalTreeParser tree1Iter = new CanonicalTreeParser(); + ObjectId tree1 = resolveCommit(commit1).getTree(); + tree1Iter.reset(reader, tree1); + + CanonicalTreeParser tree2Iter = new CanonicalTreeParser(); + ObjectId tree2 = resolveCommit(commit2).getTree(); + tree2Iter.reset(reader, tree2); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + if (StringUtils.isNotEmpty(path)) { + diffFormatter.setPathFilter(PathFilter.create(path)); + } + + return diffFormatter.scan(tree1, tree2).stream() + .map(diffEntry -> { + Diff diff = new Diff(); + try { + out.reset(); + diffFormatter.format(diffEntry); + diff.setDiffStringValue(out.toString()); + diff.setFilePath(diffEntry.getChangeType() != DiffEntry.ChangeType.DELETE ? diffEntry.getNewPath() : diffEntry.getOldPath()); + diff.setChangeType(diffEntry.getChangeType()); + try { + diff.setFileContentAtCommit1(getFileContentAtCommit(diff.getFilePath(), commit1)); + } catch (IllegalArgumentException ignored) { + } + try { + diff.setFileContentAtCommit2(getFileContentAtCommit(diff.getFilePath(), commit2)); + } catch (IllegalArgumentException ignored) { + } + return diff; + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + + private Commit toCommit(RevCommit revCommit) { + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), + revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); + } + + private RevCommit resolveCommit(String id) throws IOException { + return git.getRepository().parseCommit(resolve(id)); + } + + private ObjectId resolve(String rev) throws IOException { + return git.getRepository().resolve(rev); + } + + private , T> T execute(C command) throws GitAPIException { + if (command instanceof TransportCommand) { + configureTransportCommand((TransportCommand) command, credentialsProvider, sshSessionFactory); + } + return command.call(); + } + + private static Function> revCommitComparatorFunction = pageLink -> { + SortOrder sortOrder = pageLink.getSortOrder(); + if (sortOrder != null + && sortOrder.getProperty().equals("timestamp") + && SortOrder.Direction.ASC.equals(sortOrder.getDirection())) { + return Comparator.comparingInt(RevCommit::getCommitTime); + } + return null; + }; + + private static PageData iterableToPageData(Iterable iterable, + Function mapper, + PageLink pageLink, + Function> comparatorFunction) { + iterable = Streams.stream(iterable).collect(Collectors.toList()); + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + if (comparatorFunction != null) { + Comparator comparator = comparatorFunction.apply(pageLink); + if (comparator != null) { + iterable = Ordering.from(comparator).immutableSortedCopy(iterable); + } + } + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { + if (credentialsProvider != null) { + transportCommand.setCredentialsProvider(credentialsProvider); + } + if (sshSessionFactory != null) { + transportCommand.setTransportConfigCallback(transport -> { + if (transport instanceof SshTransport) { + SshTransport sshTransport = (SshTransport) transport; + sshTransport.setSshSessionFactory(sshSessionFactory); + } + }); + } + } + + private static CredentialsProvider newCredentialsProvider(String username, String password) { + return new UsernamePasswordCredentialsProvider(username, password == null ? "" : password); + } + + private static SshdSessionFactory newSshdSessionFactory(String privateKey, String password, File directory) { + SshdSessionFactory sshSessionFactory = null; + if (StringUtils.isNotBlank(privateKey)) { + Iterable keyPairs = loadKeyPairs(privateKey, password); + sshSessionFactory = new SshdSessionFactoryBuilder() + .setPreferredAuthentications("publickey") + .setDefaultKeysProvider(file -> keyPairs) + .setHomeDirectory(directory) + .setSshDirectory(directory) + .setServerKeyDatabase((file, file2) -> new ServerKeyDatabase() { + @Override + public List lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) { + return Collections.emptyList(); + } + + @Override + public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) { + return true; + } + }) + .build(new JGitKeyCache()); + } + return sshSessionFactory; + } + + private static Iterable loadKeyPairs(String privateKeyContent, String password) { + Iterable keyPairs = null; + try { + keyPairs = SecurityUtils.loadKeyPairIdentities(null, + null, new ByteArrayInputStream(privateKeyContent.getBytes()), (session, resourceKey, retryIndex) -> password); + } catch (Exception e) {} + if (keyPairs == null) { + throw new IllegalArgumentException("Failed to load ssh private key"); + } + return keyPairs; + } + + private static class NoMergesAndCommitMessageFilter extends RevFilter { + + private final String textSearch; + + NoMergesAndCommitMessageFilter(String textSearch) { + this.textSearch = textSearch.toLowerCase(); + } + + @Override + public boolean include(RevWalk walker, RevCommit c) { + return c.getParentCount() < 2 && c.getFullMessage().toLowerCase().contains(this.textSearch); + } + + @Override + public RevFilter clone() { + return this; + } + + @Override + public boolean requiresCommitBody() { + return false; + } + + @Override + public String toString() { + return "NO_MERGES_AND_COMMIT_MESSAGE"; + } + } + + @Data + public static class Commit { + private final long timestamp; + private final String id; + private final String message; + private final String authorName; + private final String authorEmail; + } + + @Data + public static class Status { + private final Set added; + private final Set modified; + private final Set removed; + } + + @Data + public static class Diff { + private String filePath; + private DiffEntry.ChangeType changeType; + private String fileContentAtCommit1; + private String fileContentAtCommit2; + private String diffStringValue; + } + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..057fa9c1ab --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +public interface GitRepositoryService { + + Set getActiveRepositoryTenants(); + + void prepareCommit(PendingCommit pendingCommit); + + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; + + void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception; + + void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception; + + RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception; + + void clearRepository(TenantId tenantId) throws IOException; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void cleanUp(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + + List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException; + + String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException; + + void fetch(TenantId tenantId) throws GitAPIException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java new file mode 100644 index 0000000000..ccd5fc685e --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +public class PendingCommit { + + private final UUID txId; + private final String nodeId; + private final TenantId tenantId; + private final String workingBranch; + private String branch; + private String versionName; + + private String authorName; + + private String authorEmail; + + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { + this.tenantId = tenantId; + this.nodeId = nodeId; + this.txId = txId; + this.branch = branch; + this.versionName = versionName; + this.authorName = authorName; + this.authorEmail = authorEmail; + this.workingBranch = txId.toString(); + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java new file mode 100644 index 0000000000..26ab0d94c1 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; + +import java.util.UUID; + +@RequiredArgsConstructor +@Data +public class VersionControlRequestCtx { + private final String nodeId; + private final UUID requestId; + private final TenantId tenantId; + private final RepositorySettings settings; + + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, RepositorySettings settings) { + this.nodeId = msg.getNodeId(); + this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); + this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + this.settings = settings; + } + + @Override + public String toString() { + return "VersionControlRequestCtx{" + + "nodeId='" + nodeId + '\'' + + ", requestId=" + requestId + + ", tenantId=" + tenantId + + '}'; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 932a6e5ec1..663c19348d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import java.util.Collection; @@ -36,8 +37,12 @@ public interface Dao { T save(TenantId tenantId, T t); + T saveAndFlush(TenantId tenantId, T t); + boolean removeById(TenantId tenantId, UUID id); void removeAllByIds(Collection ids); + default EntityType getEntityType() { return null; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 4f250c47c8..4be3f8e6ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -31,6 +31,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; public abstract class DaoUtil { @@ -50,7 +52,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap()); } - public static Pageable toPageable(PageLink pageLink, Map columnMap) { + public static Pageable toPageable(PageLink pageLink, Map columnMap) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(pageLink.getSortOrder(), columnMap)); } @@ -58,7 +60,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } - public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(sortOrders, columnMap)); } @@ -107,4 +109,18 @@ public abstract class DaoUtil { return ids; } + public static void processInBatches(Function> finder, int batchSize, Consumer processor) { + PageLink pageLink = new PageLink(batchSize); + PageData batch; + + boolean hasNextBatch; + do { + batch = finder.apply(pageLink); + batch.getData().forEach(processor); + + hasNextBatch = batch.hasNext(); + pageLink = pageLink.nextPageLink(); + } while (hasNextBatch); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java new file mode 100644 index 0000000000..77800c31e8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; + +import java.util.UUID; + +public interface ExportableEntityDao> extends Dao { + + T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + + default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } + + PageData findByTenantId(UUID tenantId, PageLink pageLink); + + I getExternalIdByInternal(I internalId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java new file mode 100644 index 0000000000..6e2d2e115a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import java.util.UUID; + +public interface ExportableEntityRepository { + + D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 0432e69bca..e3c51b48a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -19,10 +19,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -33,7 +35,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 729c923e54..eb85f70687 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -23,8 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; @@ -53,7 +51,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; @@ -125,7 +122,7 @@ public class BaseAssetService extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(asset.getTenantId(), assetId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete asset that has entity views!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for assetId [{}]", assetId, e); - throw new RuntimeException("Exception while finding entity views for assetId [" + assetId + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(asset.getTenantId(), assetId); + if (entityViews != null && !entityViews.isEmpty()) { + throw new DataValidationException("Can't delete asset that has entity views!"); } publishEvictEvent(new AssetCacheEvictEvent(asset.getTenantId(), asset.getName(), null)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index d79537e564..0b2abcf517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.customer; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Optional; @@ -28,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object @@ -37,7 +39,7 @@ public interface CustomerDao extends Dao, TenantEntityDao { * @return saved customer object */ Customer save(TenantId tenantId, Customer customer); - + /** * Find customers by tenant id and page link. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index c79ec44880..547589169e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -64,9 +63,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom @Autowired private DeviceService deviceService; - @Autowired - private EntityViewService entityViewService; - @Autowired private DashboardService dashboardService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index dfe9152484..4246137d9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -16,14 +16,19 @@ package org.thingsboard.server.dao.dashboard; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; +import java.util.List; +import java.util.UUID; + /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object @@ -32,4 +37,7 @@ public interface DashboardDao extends Dao, TenantEntityDao { * @return saved dashboard object */ Dashboard save(TenantId tenantId, Dashboard dashboard); + + List findByTenantIdAndTitle(UUID tenantId, String title); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index ee5c44e20f..d76a3d675e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -40,6 +40,8 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import java.util.List; + import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -56,7 +58,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb @Autowired private CustomerDao customerDao; - + @Autowired private EdgeDao edgeDao; @@ -279,19 +281,24 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb return dashboardInfoDao.findFirstByTenantIdAndName(tenantId.getId(), name); } + @Override + public List findTenantDashboardsByTitle(TenantId tenantId, String title) { + return dashboardDao.findByTenantIdAndTitle(tenantId.getId(), title); + } + private PaginatedRemover tenantDashboardsRemover = new PaginatedRemover() { - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink); - } + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, DashboardInfo entity) { - deleteDashboard(tenantId, new DashboardId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, DashboardInfo entity) { + deleteDashboard(tenantId, new DashboardId(entity.getUuidId())); + } + }; private class CustomerDashboardsUnassigner extends PaginatedRemover { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 85f66e4b67..c96f196d44 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -20,11 +20,13 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -35,7 +37,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java index bf61cbdce5..4d4c73728f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java @@ -17,14 +17,16 @@ package org.thingsboard.server.dao.device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.UUID; -public interface DeviceProfileDao extends Dao { +public interface DeviceProfileDao extends Dao, ExportableEntityDao { DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, UUID deviceProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index df5a49c1e1..95f2f034fe 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -47,8 +45,6 @@ import org.thingsboard.server.dao.service.Validator; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -76,8 +72,6 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), deviceId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete device that has entity views!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for deviceId [{}]", deviceId, e); - throw new RuntimeException("Exception while finding entity views for deviceId [" + deviceId + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(device.getTenantId(), deviceId); + if (entityViews != null && !entityViews.isEmpty()) { + throw new DataValidationException("Can't delete device that has entity views!"); } DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, deviceId); @@ -523,14 +517,9 @@ public class DeviceServiceImpl extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); - if (!CollectionUtils.isEmpty(entityViews)) { - throw new DataValidationException("Can't assign device that has entity views to another tenant!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for deviceId [{}]", device.getId(), e); - throw new RuntimeException("Exception while finding entity views for deviceId [" + device.getId() + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(device.getTenantId(), device.getId()); + if (!CollectionUtils.isEmpty(entityViews)) { + throw new DataValidationException("Can't assign device that has entity views to another tenant!"); } eventService.removeEvents(device.getTenantId(), device.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index ae522d620a..8dd8da813c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -26,8 +26,6 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 9321fd4e9e..128573af86 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -84,20 +84,16 @@ public abstract class AbstractEntityService { } protected void checkAssignedEntityViewsToEdge(TenantId tenantId, EntityId entityId, EdgeId edgeId) { - try { - List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - EntityView entityView = entityViews.get(0); - // TODO: @voba - refactor this blocking operation - Boolean relationExists = relationService.checkRelation(tenantId, edgeId, entityView.getId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE).get(); - if (relationExists) { - throw new DataValidationException("Can't unassign device/asset from edge that is related to entity view and entity view is assigned to edge!"); - } + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(tenantId, entityId); + if (entityViews != null && !entityViews.isEmpty()) { + EntityView entityView = entityViews.get(0); + Boolean relationExists = relationService.checkRelation( + tenantId, edgeId, entityView.getId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE + ); + if (relationExists) { + throw new DataValidationException("Can't unassign device/asset from edge that is related to entity view and entity view is assigned to edge!"); } - } catch (Exception e) { - log.error("[{}] Exception while finding entity views for entityId [{}]", tenantId, entityId, e); - throw new RuntimeException("Exception while finding entity views for entityId [" + entityId + "]", e); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java index f44bafe937..75a9c122e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java @@ -19,10 +19,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.List; import java.util.Optional; @@ -31,7 +33,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/28/2017. */ -public interface EntityViewDao extends Dao { +public interface EntityViewDao extends Dao, ExportableEntityDao { /** * Find entity view info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 04bdb6697a..6b1d6775d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -22,8 +22,6 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; @@ -53,7 +51,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -68,7 +65,6 @@ import static org.thingsboard.server.dao.service.Validator.validateString; public class EntityViewServiceImpl extends AbstractCachedEntityService implements EntityViewService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - public static final String INCORRECT_PAGE_LINK = "Incorrect page link "; public static final String INCORRECT_CUSTOMER_ID = "Incorrect customerId "; public static final String INCORRECT_ENTITY_VIEW_ID = "Incorrect entityViewId "; public static final String INCORRECT_EDGE_ID = "Incorrect edgeId "; @@ -280,6 +276,17 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService new EntityViewCacheValue(null, v), true)); } + @Override + public List findEntityViewsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findEntityViewsByTenantIdAndEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateId(entityId.getId(), "Incorrect entityId" + entityId); + + return cache.getAndPutInTransaction(EntityViewCacheKey.byEntityId(tenantId, entityId), + () -> entityViewDao.findEntityViewsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()), + EntityViewCacheValue::getEntityViews, v -> new EntityViewCacheValue(null, v), true); + } + @Override public void deleteEntityView(TenantId tenantId, EntityViewId entityViewId) { log.trace("Executing deleteEntityView [{}]", entityViewId); @@ -320,15 +327,10 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService implements BaseEntity { @Column(name = ModelConstants.ID_PROPERTY, columnDefinition = "uuid") protected UUID id; - @Column(name = ModelConstants.CREATED_TIME_PROPERTY) + @Column(name = ModelConstants.CREATED_TIME_PROPERTY, updatable = false) protected long createdTime; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f6f838eb62..80e21a3fae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -90,6 +90,8 @@ public class ModelConstants { * Cassandra admin_settings constants. */ public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings"; + + public static final String ADMIN_SETTINGS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key"; public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value"; @@ -559,6 +561,8 @@ public class ModelConstants { public static final String EDGE_EVENT_BY_ID_VIEW_NAME = "edge_event_by_id"; + public static final String EXTERNAL_ID_PROPERTY = "external_id"; + /** * User auth settings constants. * */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index 4405768f7c..0936ca2855 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -38,6 +38,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ASSET_LABEL_PROPER import static org.thingsboard.server.dao.model.ModelConstants.ASSET_NAME_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.EXTERNAL_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.SEARCH_TEXT_PROPERTY; @Data @@ -68,6 +69,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity @Column(name = ModelConstants.ASSET_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = EXTERNAL_ID_PROPERTY) + private UUID externalId; + public AbstractAssetEntity() { super(); } @@ -87,6 +91,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.type = asset.getType(); this.label = asset.getLabel(); this.additionalInfo = asset.getAdditionalInfo(); + if (asset.getExternalId() != null) { + this.externalId = asset.getExternalId().getId(); + } } public AbstractAssetEntity(AssetEntity assetEntity) { @@ -99,6 +106,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.label = assetEntity.getLabel(); this.searchText = assetEntity.getSearchText(); this.additionalInfo = assetEntity.getAdditionalInfo(); + this.externalId = assetEntity.getExternalId(); } @Override @@ -128,6 +136,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity asset.setType(type); asset.setLabel(label); asset.setAdditionalInfo(additionalInfo); + if (externalId != null) { + asset.setExternalId(new AssetId(externalId)); + } return asset; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 717ba1f9e3..dbc0a057ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -84,6 +84,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti @Column(name = ModelConstants.DEVICE_DEVICE_DATA_PROPERTY, columnDefinition = "jsonb") private JsonNode deviceData; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY, columnDefinition = "uuid") + private UUID externalId; + public AbstractDeviceEntity() { super(); } @@ -113,6 +116,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.type = device.getType(); this.label = device.getLabel(); this.additionalInfo = device.getAdditionalInfo(); + if (device.getExternalId() != null) { + this.externalId = device.getExternalId().getId(); + } } public AbstractDeviceEntity(DeviceEntity deviceEntity) { @@ -129,6 +135,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.additionalInfo = deviceEntity.getAdditionalInfo(); this.firmwareId = deviceEntity.getFirmwareId(); this.softwareId = deviceEntity.getSoftwareId(); + this.externalId = deviceEntity.getExternalId(); } @Override @@ -164,6 +171,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti device.setType(type); device.setLabel(label); device.setAdditionalInfo(additionalInfo); + if (externalId != null) { + device.setExternalId(new DeviceId(externalId)); + } return device; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index a8a31cd892..cc68adb2a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -89,6 +89,9 @@ public abstract class AbstractEntityViewEntity extends Bas @Column(name = ModelConstants.ENTITY_VIEW_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + private static final ObjectMapper mapper = new ObjectMapper(); public AbstractEntityViewEntity() { @@ -121,6 +124,9 @@ public abstract class AbstractEntityViewEntity extends Bas this.endTs = entityView.getEndTimeMs(); this.searchText = entityView.getSearchText(); this.additionalInfo = entityView.getAdditionalInfo(); + if (entityView.getExternalId() != null) { + this.externalId = entityView.getExternalId().getId(); + } } public AbstractEntityViewEntity(EntityViewEntity entityViewEntity) { @@ -137,6 +143,7 @@ public abstract class AbstractEntityViewEntity extends Bas this.endTs = entityViewEntity.getEndTs(); this.searchText = entityViewEntity.getSearchText(); this.additionalInfo = entityViewEntity.getAdditionalInfo(); + this.externalId = entityViewEntity.getExternalId(); } @Override @@ -172,6 +179,9 @@ public abstract class AbstractEntityViewEntity extends Bas entityView.setStartTimeMs(startTs); entityView.setEndTimeMs(endTs); entityView.setAdditionalInfo(additionalInfo); + if (externalId != null) { + entityView.setExternalId(new EntityViewId(externalId)); + } return entityView; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 49d5002af2..4da17d2c34 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -22,14 +22,18 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; +import java.util.UUID; + import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_JSON_VALUE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY_PROPERTY; @@ -41,6 +45,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY @Table(name = ADMIN_SETTINGS_COLUMN_FAMILY_NAME) public final class AdminSettingsEntity extends BaseSqlEntity implements BaseEntity { + @Column(name = ModelConstants.ADMIN_SETTINGS_TENANT_ID_PROPERTY) + private UUID tenantId; + @Column(name = ADMIN_SETTINGS_KEY_PROPERTY) private String key; @@ -57,6 +64,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl this.setUuid(adminSettings.getId().getId()); } this.setCreatedTime(adminSettings.getCreatedTime()); + this.tenantId = adminSettings.getTenantId().getId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -65,6 +73,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettings toData() { AdminSettings adminSettings = new AdminSettings(new AdminSettingsId(id)); adminSettings.setCreatedTime(createdTime); + adminSettings.setTenantId(TenantId.fromUUID(tenantId)); adminSettings.setKey(key); adminSettings.setJsonValue(jsonValue); return adminSettings; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java index 61b21ecc4f..3dec26afd9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java @@ -77,6 +77,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Column(name = ModelConstants.CUSTOMER_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public CustomerEntity() { super(); } @@ -97,6 +100,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea this.phone = customer.getPhone(); this.email = customer.getEmail(); this.additionalInfo = customer.getAdditionalInfo(); + if (customer.getExternalId() != null) { + this.externalId = customer.getExternalId().getId(); + } } @Override @@ -124,6 +130,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea customer.setPhone(phone); customer.setEmail(email); customer.setAdditionalInfo(additionalInfo); + if (externalId != null) { + customer.setExternalId(new CustomerId(externalId)); + } return customer; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index 45b4bf9210..ccaea5524e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -78,6 +78,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Column(name = ModelConstants.DASHBOARD_CONFIGURATION_PROPERTY) private JsonNode configuration; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DashboardEntity() { super(); } @@ -102,6 +105,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S this.mobileHide = dashboard.isMobileHide(); this.mobileOrder = dashboard.getMobileOrder(); this.configuration = dashboard.getConfiguration(); + if (dashboard.getExternalId() != null) { + this.externalId = dashboard.getExternalId().getId(); + } } @Override @@ -133,6 +139,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S dashboard.setMobileHide(mobileHide); dashboard.setMobileOrder(mobileOrder); dashboard.setConfiguration(configuration); + if (externalId != null) { + dashboard.setExternalId(new DashboardId(externalId)); + } return dashboard; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java index 9dc5c9780e..db1a04f746 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java @@ -104,6 +104,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl @Column(name = ModelConstants.DEVICE_PROFILE_SOFTWARE_ID_PROPERTY) private UUID softwareId; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DeviceProfileEntity() { super(); } @@ -140,6 +143,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (deviceProfile.getSoftwareId() != null) { this.softwareId = deviceProfile.getSoftwareId().getId(); } + if (deviceProfile.getExternalId() != null) { + this.externalId = deviceProfile.getExternalId().getId(); + } } @Override @@ -189,6 +195,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (softwareId != null) { deviceProfile.setSoftwareId(new OtaPackageId(softwareId)); } + if (externalId != null) { + deviceProfile.setExternalId(new DeviceProfileId(externalId)); + } return deviceProfile; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java index 09ed75b5e5..88c8c79fdc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java @@ -75,6 +75,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleChainEntity() { } @@ -94,6 +97,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT this.debugMode = ruleChain.isDebugMode(); this.configuration = ruleChain.getConfiguration(); this.additionalInfo = ruleChain.getAdditionalInfo(); + if (ruleChain.getExternalId() != null) { + this.externalId = ruleChain.getExternalId().getId(); + } } @Override @@ -120,6 +126,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT ruleChain.setDebugMode(debugMode); ruleChain.setConfiguration(configuration); ruleChain.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleChain.setExternalId(new RuleChainId(externalId)); + } return ruleChain; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index 04b7fd4ecb..70e8497c31 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -64,6 +64,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Column(name = ModelConstants.DEBUG_MODE) private boolean debugMode; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleNodeEntity() { } @@ -81,6 +84,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex this.searchText = ruleNode.getName(); this.configuration = ruleNode.getConfiguration(); this.additionalInfo = ruleNode.getAdditionalInfo(); + if (ruleNode.getExternalId() != null) { + this.externalId = ruleNode.getExternalId().getId(); + } } @Override @@ -105,6 +111,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex ruleNode.setDebugMode(debugMode); ruleNode.setConfiguration(configuration); ruleNode.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleNode.setExternalId(new RuleNodeId(externalId)); + } return ruleNode; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 5887c6f61e..cbd6068d7f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -54,6 +54,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl @Column(name = ModelConstants.WIDGETS_BUNDLE_DESCRIPTION) private String description; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public WidgetsBundleEntity() { super(); } @@ -70,6 +73,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl this.title = widgetsBundle.getTitle(); this.image = widgetsBundle.getImage(); this.description = widgetsBundle.getDescription(); + if (widgetsBundle.getExternalId() != null) { + this.externalId = widgetsBundle.getExternalId().getId(); + } } @Override @@ -93,6 +99,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl widgetsBundle.setTitle(title); widgetsBundle.setImage(image); widgetsBundle.setDescription(description); + if (externalId != null) { + widgetsBundle.setExternalId(new WidgetsBundleId(externalId)); + } return widgetsBundle; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 4d0bb18a43..1587aa9f52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -22,8 +22,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.OtaPackage; @@ -42,8 +40,6 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; import java.util.Optional; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -248,8 +244,4 @@ public class BaseOtaPackageService extends AbstractCachedEntityService toOtaPackageInfoKey(OtaPackageId otaPackageId) { - return Collections.singletonList(otaPackageId); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 912a3393c4..f60920a31d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.relation; import com.google.common.base.Function; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -24,12 +25,11 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.dao.ConcurrencyFailureException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.util.StringUtils; import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -90,7 +90,14 @@ public class BaseRelationService implements RelationService { } @Override - public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + log.trace("Executing checkRelationAsync [{}][{}][{}][{}]", from, to, relationType, typeGroup); + validate(from, to, relationType, typeGroup); + return relationDao.checkRelationAsync(tenantId, from, to, relationType, typeGroup); + } + + @Override + public boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing checkRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); return relationDao.checkRelation(tenantId, from, to, relationType, typeGroup); @@ -119,6 +126,20 @@ public class BaseRelationService implements RelationService { return result; } + @Override + public void saveRelations(TenantId tenantId, List relations) { + log.trace("Executing saveRelations [{}]", relations); + for (EntityRelation relation : relations) { + validate(relation); + } + for (List partition : Lists.partition(relations, 1024)) { + relationDao.saveRelations(tenantId, partition); + } + for (EntityRelation relation : relations) { + publishEvictEvent(EntityRelationEvent.from(relation)); + } + } + @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { log.trace("Executing saveRelationAsync [{}]", relation); @@ -167,68 +188,33 @@ public class BaseRelationService implements RelationService { return future; } + @Transactional @Override public void deleteEntityRelations(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteEntityRelations [{}]", entityId); validate(entityId); - List inboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelations.addAll(relationDao.findAllByTo(tenantId, entityId, typeGroup)); - } - - List outboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelations.addAll(relationDao.findAllByFrom(tenantId, entityId, typeGroup)); - } - - for (EntityRelation relation : inboundRelations) { - delete(tenantId, relation, true); - } + List inboundRelations = new ArrayList<>(relationDao.findAllByTo(tenantId, entityId)); + List outboundRelations = new ArrayList<>(relationDao.findAllByFrom(tenantId, entityId)); - for (EntityRelation relation : outboundRelations) { - delete(tenantId, relation, false); - } - - relationDao.deleteOutboundRelations(tenantId, entityId); - - } + if (!inboundRelations.isEmpty()) { + try { + relationDao.deleteInboundRelations(tenantId, entityId); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", inboundRelations, e); + } - @Override - public ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entityId) { - log.trace("Executing deleteEntityRelationsAsync [{}]", entityId); - validate(entityId); - List>> inboundRelationsList = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelationsList.add(executor.submit(() -> relationDao.findAllByTo(tenantId, entityId, typeGroup))); + for (EntityRelation relation : inboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } } - ListenableFuture>> inboundRelations = Futures.allAsList(inboundRelationsList); + if (!outboundRelations.isEmpty()) { + relationDao.deleteOutboundRelations(tenantId, entityId); - List>> outboundRelationsList = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelationsList.add(executor.submit(() -> relationDao.findAllByFrom(tenantId, entityId, typeGroup))); + for (EntityRelation relation : outboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } } - - ListenableFuture>> outboundRelations = Futures.allAsList(outboundRelationsList); - - ListenableFuture> inboundDeletions = Futures.transformAsync(inboundRelations, - relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, true); - return Futures.allAsList(results); - }, MoreExecutors.directExecutor()); - - ListenableFuture> outboundDeletions = Futures.transformAsync(outboundRelations, - relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, false); - return Futures.allAsList(results); - }, MoreExecutors.directExecutor()); - - ListenableFuture>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions); - - return Futures.transform(Futures.transformAsync(deletionsFuture, - (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId), - MoreExecutors.directExecutor()), - result -> null, MoreExecutors.directExecutor()); } private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, boolean deleteFromDb) { @@ -252,18 +238,6 @@ public class BaseRelationService implements RelationService { } } - boolean delete(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { - eventPublisher.publishEvent(EntityRelationEvent.from(relation)); - if (deleteFromDb) { - try { - return relationDao.deleteRelation(tenantId, relation); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", relation, e); - } - } - return false; - } - @Override public List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { validate(from); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index e15f4f3d69..a3e81c2652 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import java.util.Collection; import java.util.List; /** @@ -31,18 +32,26 @@ public interface RelationDao { List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); + List findAllByFrom(TenantId tenantId, EntityId from); + List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + List findAllByTo(TenantId tenantId, EntityId to); + List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); - ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + + boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean saveRelation(TenantId tenantId, EntityRelation relation); + void saveRelations(TenantId tenantId, Collection relations); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); boolean deleteRelation(TenantId tenantId, EntityRelation relation); @@ -53,7 +62,9 @@ public interface RelationDao { ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean deleteOutboundRelations(TenantId tenantId, EntityId entity); + void deleteOutboundRelations(TenantId tenantId, EntityId entity); + + void deleteInboundRelations(TenantId tenantId, EntityId entity); ListenableFuture deleteOutboundRelationsAsync(TenantId tenantId, EntityId entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 4be7227214..28aa798c53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -58,6 +58,8 @@ import org.thingsboard.server.dao.service.Validator; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -68,7 +70,9 @@ import java.util.Set; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.TENANT; -import static org.thingsboard.server.dao.service.Validator.*; +import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; +import static org.thingsboard.server.dao.service.Validator.validateString; /** * Created by igor on 3/12/18. @@ -139,6 +143,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List nodes = ruleChainMetaData.getNodes(); List toAddOrUpdate = new ArrayList<>(); List toDelete = new ArrayList<>(); + List relations = new ArrayList<>(); Map ruleNodeIndexMap = new HashMap<>(); if (nodes != null) { @@ -169,15 +174,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (RuleNode node : toAddOrUpdate) { node.setRuleChainId(ruleChain.getId()); RuleNode savedNode = ruleNodeDao.save(tenantId, node); - createRelation(tenantId, new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), + relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); int index = nodes.indexOf(node); nodes.set(index, savedNode); ruleNodeIndexMap.put(savedNode.getId(), index); } } - for (RuleNode node : toDelete) { - deleteRuleNode(tenantId, node.getId()); + if (!toDelete.isEmpty()) { + deleteRuleNodes(tenantId, toDelete); } RuleNodeId firstRuleNodeId = null; if (nodes != null) { @@ -194,7 +199,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC EntityId from = nodes.get(nodeConnection.getFromIndex()).getId(); EntityId to = nodes.get(nodeConnection.getToIndex()).getId(); String type = nodeConnection.getType(); - createRelation(tenantId, new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); + relations.add(new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); } } if (ruleChainMetaData.getRuleChainConnections() != null) { @@ -220,7 +225,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleChainToRuleNode.setTo(targetNode.getId()); sourceRuleChainToRuleNode.setType(EntityRelation.CONTAINS_TYPE); sourceRuleChainToRuleNode.setTypeGroup(RelationTypeGroup.RULE_CHAIN); - relationService.saveRelation(tenantId, sourceRuleChainToRuleNode); + relations.add(sourceRuleChainToRuleNode); EntityRelation sourceRuleNodeToTargetRuleNode = new EntityRelation(); EntityId from = nodes.get(nodeToRuleChainConnection.getFromIndex()).getId(); @@ -228,11 +233,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleNodeToTargetRuleNode.setTo(targetNode.getId()); sourceRuleNodeToTargetRuleNode.setType(nodeToRuleChainConnection.getType()); sourceRuleNodeToTargetRuleNode.setTypeGroup(RelationTypeGroup.RULE_NODE); - relationService.saveRelation(tenantId, sourceRuleNodeToTargetRuleNode); - } + relations.add(sourceRuleNodeToTargetRuleNode); + } } } + if (!relations.isEmpty()) { + relationService.saveRelations(tenantId, relations); + } + return RuleChainUpdateResult.successful(updatedRuleNodes); } @@ -271,6 +280,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleChainMetaData ruleChainMetaData = new RuleChainMetaData(); ruleChainMetaData.setRuleChainId(ruleChainId); List ruleNodes = getRuleChainNodes(tenantId, ruleChainId); + Collections.sort(ruleNodes, Comparator.comparingLong(RuleNode::getCreatedTime).thenComparing(RuleNode::getId, Comparator.comparing(RuleNodeId::getId))); Map ruleNodeIndexMap = new HashMap<>(); for (RuleNode node : ruleNodes) { ruleNodeIndexMap.put(node.getId(), ruleNodes.indexOf(node)); @@ -293,6 +303,10 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } } } + if (ruleChainMetaData.getConnections() != null) { + Collections.sort(ruleChainMetaData.getConnections(), Comparator.comparingInt(NodeConnectionInfo::getFromIndex) + .thenComparing(NodeConnectionInfo::getToIndex).thenComparing(NodeConnectionInfo::getType)); + } return ruleChainMetaData; } @@ -390,6 +404,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return ruleChainDao.findRuleChainsByTenantIdAndType(tenantId.getId(), type, pageLink); } + @Override + public Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name) { + return ruleChainDao.findByTenantIdAndTypeAndName(tenantId, type, name); + } + @Override @Transactional public void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId) { @@ -456,7 +475,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC ruleChain.setRoot(false); if (overwrite) { - Collection existingRuleChains = ruleChainDao.findByTenantIdAndTypeAndName(tenantId, + Collection existingRuleChains = findTenantRuleChainsByTypeAndName(tenantId, Optional.ofNullable(ruleChain.getType()).orElse(RuleChainType.CORE), ruleChain.getName()); Optional existingRuleChain = existingRuleChains.stream().findFirst(); if (existingRuleChain.isPresent()) { @@ -699,6 +718,19 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC throw t; } } + deleteRuleNodes(tenantId, ruleChainId); + } + + private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { + List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + for (var node : ruleNodes) { + deleteEntityRelations(tenantId, node.getId()); + } + ruleNodeDao.deleteByIdIn(ruleNodeIds); + } + + @Override + public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { List nodeRelations = getRuleChainToNodeRelations(tenantId, ruleChainId); for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index f53f8a9e9a..d19074aa53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Collection; @@ -29,7 +31,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java index b4f087b98a..4fb15e5721 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,4 +33,8 @@ public interface RuleNodeDao extends Dao { List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search); PageData findAllRuleNodesByType(String type, PageLink pageLink); + + List findByExternalIds(RuleChainId ruleChainId, List externalIds); + + void deleteByIdIn(List ruleNodeIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java index 5bbe636dc7..a95f7a43aa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java @@ -18,18 +18,17 @@ package org.thingsboard.server.dao.service.validator; import lombok.AllArgsConstructor; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component @AllArgsConstructor public class AlarmDataValidator extends DataValidator { - private final TenantDao tenantDao; + private final TenantService tenantService; @Override protected void validateDataImpl(TenantId tenantId, Alarm alarm) { @@ -48,8 +47,7 @@ public class AlarmDataValidator extends DataValidator { if (alarm.getTenantId() == null) { throw new DataValidationException("Alarm should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(alarm.getTenantId(), alarm.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(alarm.getTenantId())) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java index a89acb9631..28e4dc0582 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java @@ -15,29 +15,29 @@ */ package org.thingsboard.server.dao.service.validator; -import lombok.AllArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component -@AllArgsConstructor public class ApiUsageDataValidator extends DataValidator { - private final TenantDao tenantDao; + @Lazy + @Autowired + private TenantService tenantService; @Override protected void validateDataImpl(TenantId requestTenantId, ApiUsageState apiUsageState) { if (apiUsageState.getTenantId() == null) { throw new DataValidationException("ApiUsageState should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(requestTenantId, apiUsageState.getTenantId().getId()); - if (tenant == null && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { + if (!tenantService.tenantExists(apiUsageState.getTenantId()) && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { throw new DataValidationException("ApiUsageState is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java index a2204491f9..1f303cc79d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,7 +31,7 @@ import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -43,7 +42,7 @@ public class AssetDataValidator extends DataValidator { private AssetDao assetDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -82,8 +81,7 @@ public class AssetDataValidator extends DataValidator { if (asset.getTenantId() == null) { throw new DataValidationException("Asset should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, asset.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(asset.getTenantId())) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java index 8184ff8e8e..c7ffe8c61e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java @@ -20,18 +20,17 @@ import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.Objects; public abstract class BaseOtaPackageDataValidator> extends DataValidator { @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private DeviceProfileDao deviceProfileDao; @@ -40,8 +39,7 @@ public abstract class BaseOtaPackageDataValidator> extends if (otaPackageInfo.getTenantId() == null) { throw new DataValidationException("OtaPackage should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(otaPackageInfo.getTenantId(), otaPackageInfo.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(otaPackageInfo.getTenantId())) { throw new DataValidationException("OtaPackage is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java index b8e1067887..9cc8d6a0de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.customer.CustomerDao; @@ -29,7 +28,7 @@ import org.thingsboard.server.dao.customer.CustomerServiceImpl; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.Optional; @@ -40,7 +39,7 @@ public class CustomerDataValidator extends DataValidator { private CustomerDao customerDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -87,8 +86,7 @@ public class CustomerDataValidator extends DataValidator { if (customer.getTenantId() == null) { throw new DataValidationException("Customer should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, customer.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(customer.getTenantId())) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java index ee226af853..23b960253c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java @@ -21,14 +21,13 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component public class DashboardDataValidator extends DataValidator { @@ -37,7 +36,7 @@ public class DashboardDataValidator extends DataValidator { private DashboardDao dashboardDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -59,8 +58,7 @@ public class DashboardDataValidator extends DataValidator { if (dashboard.getTenantId() == null) { throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, dashboard.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(dashboard.getTenantId())) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java index e9d8b82e1c..23e520f070 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +34,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.Optional; @@ -48,7 +47,7 @@ public class DeviceDataValidator extends DataValidator { private DeviceDao deviceDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -85,8 +84,7 @@ public class DeviceDataValidator extends DataValidator { if (device.getTenantId() == null) { throw new DataValidationException("Device should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(device.getTenantId(), device.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(device.getTenantId())) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index f04cbd0226..429a328c53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -67,7 +66,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.HashSet; import java.util.List; @@ -90,7 +89,7 @@ public class DeviceProfileDataValidator extends DataValidator { @Autowired private DeviceDao deviceDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy private QueueService queueService; @@ -119,8 +118,7 @@ public class DeviceProfileDataValidator extends DataValidator { if (deviceProfile.getTenantId() == null) { throw new DataValidationException("Device profile should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(deviceProfile.getTenantId(), deviceProfile.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(deviceProfile.getTenantId())) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java index 8289d21b23..b88724f7af 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java @@ -19,7 +19,6 @@ import lombok.AllArgsConstructor; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +26,7 @@ import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.edge.EdgeDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -36,7 +35,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public class EdgeDataValidator extends DataValidator { private final EdgeDao edgeDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final CustomerDao customerDao; @Override @@ -65,8 +64,7 @@ public class EdgeDataValidator extends DataValidator { if (edge.getTenantId() == null) { throw new DataValidationException("Edge should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(edge.getTenantId(), edge.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(edge.getTenantId())) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java index a38850c63c..7532dda195 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java @@ -20,14 +20,13 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -36,7 +35,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public class EntityViewDataValidator extends DataValidator { private final EntityViewDao entityViewDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final CustomerDao customerDao; @Override @@ -69,8 +68,7 @@ public class EntityViewDataValidator extends DataValidator { if (entityView.getTenantId() == null) { throw new DataValidationException("Entity view should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, entityView.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(entityView.getTenantId())) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java index 90ac54577a..49de43dba3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; @@ -28,7 +27,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.common.data.EntityType.TB_RESOURCE; @@ -39,7 +38,7 @@ public class ResourceDataValidator extends DataValidator { private TbResourceDao resourceDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -73,8 +72,7 @@ public class ResourceDataValidator extends DataValidator { resource.setTenantId(TenantId.fromUUID(ModelConstants.NULL_UUID)); } if (!resource.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, resource.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(resource.getTenantId())) { throw new DataValidationException("Resource is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java index f36b4d23fa..1581a61b80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -30,7 +29,7 @@ import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component public class RuleChainDataValidator extends DataValidator { @@ -43,7 +42,7 @@ public class RuleChainDataValidator extends DataValidator { private RuleChainService ruleChainService; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -68,8 +67,7 @@ public class RuleChainDataValidator extends DataValidator { if (ruleChain.getTenantId() == null || ruleChain.getTenantId().isNullUid()) { throw new DataValidationException("Rule chain should be assigned to tenant!"); } - Tenant tenant = tenantDao.findById(tenantId, ruleChain.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(ruleChain.getTenantId())) { throw new DataValidationException("Rule chain is referencing to non-existent tenant!"); } if (ruleChain.isRoot() && RuleChainType.CORE.equals(ruleChain.getType())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java index 94b894acce..21e2143f1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,7 +31,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserDao; import org.thingsboard.server.dao.user.UserService; @@ -46,9 +45,6 @@ public class UserDataValidator extends DataValidator { @Lazy private UserService userService; - @Autowired - private TenantDao tenantDao; - @Autowired private CustomerDao customerDao; @@ -56,6 +52,10 @@ public class UserDataValidator extends DataValidator { @Lazy private TbTenantProfileCache tenantProfileCache; + @Autowired + @Lazy + private TenantService tenantService; + @Override protected void validateCreate(TenantId tenantId, User user) { if (!user.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { @@ -119,8 +119,7 @@ public class UserDataValidator extends DataValidator { + " already present in database!"); } if (!tenantId.getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, user.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(user.getTenantId())) { throw new DataValidationException("User is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java index 93a1766546..25c6b5db86 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java @@ -27,6 +27,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.widget.WidgetTypeDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; @@ -35,8 +36,8 @@ import org.thingsboard.server.dao.widget.WidgetsBundleDao; public class WidgetTypeDataValidator extends DataValidator { private final WidgetTypeDao widgetTypeDao; - private final TenantDao tenantDao; private final WidgetsBundleDao widgetsBundleDao; + private final TenantService tenantService; @Override protected void validateDataImpl(TenantId tenantId, WidgetTypeDetails widgetTypeDetails) { @@ -53,8 +54,7 @@ public class WidgetTypeDataValidator extends DataValidator { widgetTypeDetails.setTenantId(TenantId.fromUUID(ModelConstants.NULL_UUID)); } if (!widgetTypeDetails.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, widgetTypeDetails.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(widgetTypeDetails.getTenantId())) { throw new DataValidationException("Widget type is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java index b140dfc79d..b950a12826 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java @@ -18,13 +18,12 @@ package org.thingsboard.server.dao.service.validator; import lombok.AllArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.widget.WidgetsBundleDao; @Component @@ -32,7 +31,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleDao; public class WidgetsBundleDataValidator extends DataValidator { private final WidgetsBundleDao widgetsBundleDao; - private final TenantDao tenantDao; + private final TenantService tenantService; @Override protected void validateDataImpl(TenantId tenantId, WidgetsBundle widgetsBundle) { @@ -43,8 +42,7 @@ public class WidgetsBundleDataValidator extends DataValidator { widgetsBundle.setTenantId(TenantId.fromUUID(ModelConstants.NULL_UUID)); } if (!widgetsBundle.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, widgetsBundle.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(widgetsBundle.getTenantId())) { throw new DataValidationException("Widgets bundle is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index faea2cc34e..972d580cd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.UUID; + public interface AdminSettingsDao extends Dao { /** @@ -35,6 +37,10 @@ public interface AdminSettingsDao extends Dao { * @param key the key * @return the admin settings object */ - AdminSettings findByKey(TenantId tenantId, String key); + AdminSettings findByTenantIdAndKey(UUID tenantId, String key); + + boolean removeByTenantIdAndKey(UUID tenantId, String key); + + void removeByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bf4c71d296..28b4c97876 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; @@ -46,21 +47,40 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings findAdminSettingsByKey(TenantId tenantId, String key) { log.trace("Executing findAdminSettingsByKey [{}]", key); Validator.validateString(key, "Incorrect key " + key); - return adminSettingsDao.findByKey(tenantId, key); + return findAdminSettingsByTenantIdAndKey(TenantId.SYS_TENANT_ID, key); + } + + @Override + public AdminSettings findAdminSettingsByTenantIdAndKey(TenantId tenantId, String key) { + return adminSettingsDao.findByTenantIdAndKey(tenantId.getId(), key); } @Override public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if(adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { + if (adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); } } - + if (adminSettings.getTenantId() == null) { + adminSettings.setTenantId(TenantId.SYS_TENANT_ID); + } return adminSettingsDao.save(tenantId, adminSettings); } + @Override + public boolean deleteAdminSettingsByTenantIdAndKey(TenantId tenantId, String key) { + log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); + Validator.validateString(key, "Incorrect key " + key); + return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); + } + + @Override + public void deleteAdminSettingsByTenantId(TenantId tenantId) { + adminSettingsDao.removeByTenantId(tenantId.getId()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 3c996ff6f5..ad6bb5ccec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -67,6 +67,14 @@ public abstract class JpaAbstractDao, D> return DaoUtil.getData(entity); } + @Override + @Transactional + public D saveAndFlush(TenantId tenantId, D domain) { + D d = save(tenantId, domain); + getRepository().flush(); + return d; + } + @Override public D findById(TenantId tenantId, UUID key) { log.debug("Get entity by key {}", key); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index 43d43a5f8e..0b409d9256 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -16,6 +16,9 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.EntityAlarmCompositeKey; import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; @@ -28,5 +31,7 @@ public interface EntityAlarmRepository extends JpaRepository findAllByAlarmId(UUID alarmId); @Transactional - void deleteByEntityId(UUID id); + @Modifying + @Query("DELETE FROM EntityAlarmEntity e where e.entityId = :entityId") + void deleteByEntityId(@Param("entityId") UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index cdf04f47c5..8e165898a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -188,4 +189,10 @@ public class JpaAlarmDao extends JpaAbstractDao implements A log.trace("[{}] Try to delete entity alarm records using [{}]", tenantId, entityId); entityAlarmRepository.deleteByEntityId(entityId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 833fcc2de8..6d4667d8ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -public interface AssetRepository extends JpaRepository { +public interface AssetRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo) " + "FROM AssetEntity a " + @@ -143,4 +144,8 @@ public interface AssetRepository extends JpaRepository { Pageable pageable); Long countByTenantIdAndTypeIsNot(UUID tenantId, String type); + + @Query("SELECT externalId FROM AssetEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index c9752c5390..b18d6c5f51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -208,4 +209,31 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); } + + @Override + public Asset findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Asset findByTenantIdAndName(UUID tenantId, String name) { + return findAssetsByTenantIdAndName(tenantId, name).orElse(null); + } + + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findAssetsByTenantId(tenantId, pageLink); + } + + @Override + public AssetId getExternalIdByInternal(AssetId internalId) { + return Optional.ofNullable(assetRepository.getExternalIdById(internalId.getId())) + .map(AssetId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java index 1a549436d0..4ec4a562c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.plugin.ComponentScope; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; @@ -46,5 +48,8 @@ public interface ComponentDescriptorRepository extends JpaRepository { +public interface CustomerRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT c FROM CustomerEntity c WHERE c.tenantId = :tenantId " + "AND LOWER(c.searchText) LIKE LOWER(CONCAT('%', :textSearch, '%'))") @@ -38,4 +39,8 @@ public interface CustomerRepository extends JpaRepository CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); Long countByTenantId(UUID tenantId); + + @Query("SELECT externalId FROM CustomerEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 02311df61c..ab3a4b4f8d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -19,6 +19,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -68,4 +70,31 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findCustomersByTenantId(tenantId, pageLink); + } + + @Override + public CustomerId getExternalIdByInternal(CustomerId internalId) { + return Optional.ofNullable(customerRepository.getExternalIdById(internalId.getId())) + .map(CustomerId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 36518887d4..eaa247b96d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,15 +15,29 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; +import java.util.List; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface DashboardRepository extends JpaRepository { +public interface DashboardRepository extends JpaRepository, ExportableEntityRepository { Long countByTenantId(UUID tenantId); + + List findByTenantIdAndTitle(UUID tenantId, String title); + + Page findByTenantId(UUID tenantId, Pageable pageable); + + @Query("SELECT externalId FROM DashboardEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 98edcaa1fd..425c54c647 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -19,11 +19,18 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; +import java.util.List; +import java.util.Optional; import java.util.UUID; /** @@ -49,4 +56,31 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.toPageData(dashboardRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); + } + + @Override + public DashboardId getExternalIdByInternal(DashboardId internalId) { + return Optional.ofNullable(dashboardRepository.getExternalIdById(internalId.getId())) + .map(DashboardId::new).orElse(null); + } + + @Override + public List findByTenantIdAndTitle(UUID tenantId, String title) { + return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index e61a60b5b6..fea0e4bfc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -22,11 +22,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import java.util.UUID; -public interface DeviceProfileRepository extends JpaRepository { +public interface DeviceProfileRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.common.data.DeviceProfileInfo(d.id, d.name, d.image, d.defaultDashboardId, d.type, d.transportType) " + "FROM DeviceProfileEntity d " + @@ -66,4 +67,8 @@ public interface DeviceProfileRepository extends JpaRepository { +public interface DeviceRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.DeviceInfoEntity(d, c.title, c.additionalInfo, p.name) " + "FROM DeviceEntity d " + @@ -244,4 +245,8 @@ public interface DeviceRepository extends JpaRepository { "INNER JOIN DeviceProfileEntity p ON d.deviceProfileId = p.id " + "WHERE p.transportType = :transportType") Page findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType, Pageable pageable); + + @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 1d75690e50..8513e440b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageUtil; @@ -302,4 +303,31 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao Objects.toString(pageLink.getTextSearch(), ""), DaoUtil.toPageable(pageLink))); } + + @Override + public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Device findByTenantIdAndName(UUID tenantId, String name) { + return findDeviceByTenantIdAndName(tenantId, name).orElse(null); + } + + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findDevicesByTenantId(tenantId, pageLink); + } + + @Override + public DeviceId getExternalIdByInternal(DeviceId internalId) { + return Optional.ofNullable(deviceRepository.getExternalIdById(internalId.getId())) + .map(DeviceId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 80513d77c6..07a680493f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -23,6 +23,8 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -32,6 +34,7 @@ import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.Objects; +import java.util.Optional; import java.util.UUID; @Component @@ -109,4 +112,31 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findDeviceProfiles(TenantId.fromUUID(tenantId), pageLink); + } + + @Override + public DeviceProfileId getExternalIdByInternal(DeviceProfileId internalId) { + return Optional.ofNullable(deviceProfileRepository.getExternalIdById(internalId.getId())) + .map(DeviceProfileId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 4be9c08849..802c9b99b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -193,4 +193,9 @@ public class JpaEdgeDao extends JpaAbstractSearchTextDao imple return list; } + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 7a88aa3c5b..0999523c90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/31/2017. */ -public interface EntityViewRepository extends JpaRepository { +public interface EntityViewRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.EntityViewInfoEntity(e, c.title, c.additionalInfo) " + "FROM EntityViewEntity e " + @@ -139,4 +140,7 @@ public interface EntityViewRepository extends JpaRepository findByTenantId(UUID tenantId, PageLink pageLink) { + return findEntityViewsByTenantId(tenantId, pageLink); + } + + @Override + public EntityViewId getExternalIdByInternal(EntityViewId internalId) { + return Optional.ofNullable(entityViewRepository.getExternalIdById(internalId.getId())) + .map(EntityViewId::new).orElse(null); + } + + @Override + public EntityView findByTenantIdAndName(UUID tenantId, String name) { + return findEntityViewByTenantIdAndName(tenantId, name).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java index 53c1318948..6af11346fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.sql.OtaPackageEntity; @@ -48,4 +49,10 @@ public class JpaOtaPackageDao extends JpaAbstractSearchTextDao(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.WIDGETS_BUNDLE, new HashSet<>(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.API_USAGE_STATE, apiUsageStateEntityFields); + allowedEntityFieldMap.put(EntityType.DEVICE_PROFILE, Set.of(CREATED_TIME, NAME, TYPE)); entityFieldColumnMap.put(CREATED_TIME, ModelConstants.CREATED_TIME_PROPERTY); entityFieldColumnMap.put(ENTITY_TYPE, ModelConstants.ENTITY_TYPE_PROPERTY); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java deleted file mode 100644 index ec6df53ae2..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.relation; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.jpa.repository.Modifying; -import org.thingsboard.server.dao.model.sql.RelationEntity; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.Query; - -@Slf4j -public abstract class AbstractRelationInsertRepository implements RelationInsertRepository { - - @PersistenceContext - protected EntityManager entityManager; - - protected Query getQuery(RelationEntity entity, String query) { - Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); - if (entity.getAdditionalInfo() == null) { - nativeQuery.setParameter("additionalInfo", null); - } else { - nativeQuery.setParameter("additionalInfo", entity.getAdditionalInfo().toString()); - } - return nativeQuery - .setParameter("fromId", entity.getFromId()) - .setParameter("fromType", entity.getFromType()) - .setParameter("toId", entity.getToId()) - .setParameter("toType", entity.getToType()) - .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) - .setParameter("relationType", entity.getRelationType()); - } - - @Modifying - protected abstract RelationEntity processSaveOrUpdate(RelationEntity entity); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 71e36fcae1..422e0a8623 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -33,7 +33,11 @@ import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; /** * Created by Valerii Sosliuk on 5/29/2017. @@ -42,6 +46,12 @@ import java.util.List; @Component public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { + private static final List ALL_TYPE_GROUP_NAMES = new ArrayList<>(); + + static { + Arrays.stream(RelationTypeGroup.values()).map(RelationTypeGroup::name).forEach(ALL_TYPE_GROUP_NAMES::add); + } + @Autowired private RelationRepository relationRepository; @@ -57,6 +67,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByFrom(TenantId tenantId, EntityId from) { + return DaoUtil.convertDataList( + relationRepository.findAllByFromIdAndFromTypeAndRelationTypeGroupIn( + from.getId(), + from.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -76,6 +95,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByTo(TenantId tenantId, EntityId to) { + return DaoUtil.convertDataList( + relationRepository.findAllByToIdAndToTypeAndRelationTypeGroupIn( + to.getId(), + to.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -87,9 +115,14 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + return service.submit(() -> checkRelation(tenantId, from, to, relationType, typeGroup)); + } + + @Override + public boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); - return service.submit(() -> relationRepository.existsById(key)); + return relationRepository.existsById(key); } @Override @@ -112,6 +145,12 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null; } + @Override + public void saveRelations(TenantId tenantId, Collection relations) { + List entities = relations.stream().map(RelationEntity::new).collect(Collectors.toList()); + relationInsertRepository.saveOrUpdate(entities); + } + @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { return service.submit(() -> relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null); @@ -156,19 +195,21 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public boolean deleteOutboundRelations(TenantId tenantId, EntityId entity) { - boolean relationExistsBeforeDelete = false; + public void deleteOutboundRelations(TenantId tenantId, EntityId entity) { try { - relationExistsBeforeDelete = relationRepository - .findAllByFromIdAndFromType(entity.getId(), entity.getEntityType().name()) - .size() > 0; - if (relationExistsBeforeDelete) { - relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); - } + relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", entity, e); + } + } + + @Override + public void deleteInboundRelations(TenantId tenantId, EntityId entity) { + try { + relationRepository.deleteByToIdAndToTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), ALL_TYPE_GROUP_NAMES); } catch (ConcurrencyFailureException e) { log.debug("Concurrency exception while deleting relations [{}]", entity, e); } - return relationExistsBeforeDelete; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java index a07b11bcda..527a7f2bcd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java @@ -17,8 +17,12 @@ package org.thingsboard.server.dao.sql.relation; import org.thingsboard.server.dao.model.sql.RelationEntity; +import java.util.List; + public interface RelationInsertRepository { RelationEntity saveOrUpdate(RelationEntity entity); + void saveOrUpdate(List entities); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index bddd344672..d71077c55e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.relation; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; @@ -35,6 +36,10 @@ public interface RelationRepository String fromType, String relationTypeGroup); + List findAllByFromIdAndFromTypeAndRelationTypeGroupIn(UUID fromId, + String fromType, + List relationTypeGroups); + List findAllByFromIdAndFromTypeAndRelationTypeAndRelationTypeGroup(UUID fromId, String fromType, String relationType, @@ -44,6 +49,10 @@ public interface RelationRepository String toType, String relationTypeGroup); + List findAllByToIdAndToTypeAndRelationTypeGroupIn(UUID toId, + String toType, + List relationTypeGroups); + List findAllByToIdAndToTypeAndRelationTypeAndRelationTypeGroup(UUID toId, String toType, String relationType, @@ -64,6 +73,13 @@ public interface RelationRepository void deleteById(RelationCompositeKey id); @Transactional - void deleteByFromIdAndFromType(UUID fromId, String fromType); + @Modifying + @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType") + void deleteByFromIdAndFromType(@Param("fromId") UUID fromId, @Param("fromType") String fromType); + + @Transactional + @Modifying + @Query("DELETE FROM RelationEntity r where r.toId = :toId and r.toType = :toType and r.relationTypeGroup in :relationTypeGroups") + void deleteByToIdAndToTypeAndRelationTypeGroupIn(@Param("toId") UUID toId, @Param("toType") String toType, @Param("relationTypeGroups") List relationTypeGroups); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java index 23d24588b9..cbc4ca1e02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -15,25 +15,90 @@ */ package org.thingsboard.server.dao.sql.relation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.model.sql.RelationEntity; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.Query; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + @Repository @Transactional -public class SqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { +public class SqlRelationInsertRepository implements RelationInsertRepository { - private static final String INSERT_ON_CONFLICT_DO_UPDATE = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JPA = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) " + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = :additionalInfo returning *"; + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JDBC = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + " VALUES (?, ?, ?, ?, ?, ?, ?) " + + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = ?"; + + + @PersistenceContext + protected EntityManager entityManager; + + @Autowired + protected JdbcTemplate jdbcTemplate; + + protected Query getQuery(RelationEntity entity, String query) { + Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); + if (entity.getAdditionalInfo() == null) { + nativeQuery.setParameter("additionalInfo", null); + } else { + nativeQuery.setParameter("additionalInfo", JacksonUtil.toString(entity.getAdditionalInfo())); + } + return nativeQuery + .setParameter("fromId", entity.getFromId()) + .setParameter("fromType", entity.getFromType()) + .setParameter("toId", entity.getToId()) + .setParameter("toType", entity.getToType()) + .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) + .setParameter("relationType", entity.getRelationType()); + } + @Override public RelationEntity saveOrUpdate(RelationEntity entity) { - return processSaveOrUpdate(entity); + return (RelationEntity) getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE_JPA).getSingleResult(); } @Override - protected RelationEntity processSaveOrUpdate(RelationEntity entity) { - return (RelationEntity) getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE).getSingleResult(); + public void saveOrUpdate(List entities) { + jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RelationEntity relation = entities.get(i); + ps.setObject(1, relation.getFromId()); + ps.setString(2, relation.getFromType()); + ps.setObject(3, relation.getToId()); + ps.setString(4, relation.getToType()); + + ps.setString(5, relation.getRelationTypeGroup()); + ps.setString(6, relation.getRelationType()); + + if (relation.getAdditionalInfo() == null) { + ps.setString(7, null); + ps.setString(8, null); + } else { + String json = JacksonUtil.toString(relation.getAdditionalInfo()); + ps.setString(7, json); + ps.setString(8, json); + } + } + + @Override + public int getBatchSize() { + return entities.size(); + } + }); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index dbf60af096..5e4914f470 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.resource; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; @@ -96,4 +97,10 @@ public class JpaTbResourceDao extends JpaAbstractSearchTextDao implements RpcDao public Long deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + + @Override + public EntityType getEntityType() { + return EntityType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 3a7fc921e5..9ec2bab280 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,6 +33,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.Collection; import java.util.Objects; +import java.util.Optional; import java.util.UUID; @Slf4j @@ -108,4 +111,25 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId, pageLink); + } + + @Override + public RuleChainId getExternalIdByInternal(RuleChainId internalId) { + return Optional.ofNullable(ruleChainRepository.getExternalIdById(internalId.getId())) + .map(RuleChainId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index a0f84758c6..7fc1229cd3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -19,6 +19,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,6 +34,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Component @@ -63,4 +67,20 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao findByExternalIds(RuleChainId ruleChainId, List externalIds) { + return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByRuleChainIdAndExternalIdIn(ruleChainId.getId(), + externalIds.stream().map(RuleNodeId::getId).collect(Collectors.toList()))); + } + + @Override + public void deleteByIdIn(List ruleNodeIds) { + ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index fe9be8dcc7..7c4b4b86f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -21,12 +21,13 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.RuleChainEntity; import java.util.List; import java.util.UUID; -public interface RuleChainRepository extends JpaRepository { +public interface RuleChainRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT rc FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId " + "AND LOWER(rc.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") @@ -66,4 +67,7 @@ public interface RuleChainRepository extends JpaRepository findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); + @Query("SELECT externalId FROM RuleChainEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index f584ff2b02..e6a08b910a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.rule; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import java.util.List; @@ -31,12 +33,19 @@ public interface RuleNodeRepository extends JpaRepository "(select id from RuleChainEntity rc WHERE rc.tenantId = :tenantId) " + "AND r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") List findRuleNodesByTenantIdAndType(@Param("tenantId") UUID tenantId, - @Param("ruleType") String ruleType, - @Param("searchText") String searchText); + @Param("ruleType") String ruleType, + @Param("searchText") String searchText); @Query("SELECT r FROM RuleNodeEntity r WHERE r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") Page findAllRuleNodesByType(@Param("ruleType") String ruleType, @Param("searchText") String searchText, Pageable pageable); + List findRuleNodesByRuleChainIdAndExternalIdIn(UUID ruleChainId, List externalIds); + + @Transactional + @Modifying + @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") + void deleteByIdIn(@Param("ids") List ids); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 58423645b1..e607517f52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -25,5 +25,12 @@ import java.util.UUID; */ public interface AdminSettingsRepository extends JpaRepository { - AdminSettingsEntity findByKey(String key); + AdminSettingsEntity findByTenantIdAndKey(UUID tenantId, String key); + + void deleteByTenantIdAndKey(UUID tenantId, String key); + + void deleteByTenantId(UUID tenantId); + + boolean existsByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 438274e3cc..bd815bc410 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -46,7 +47,24 @@ public class JpaAdminSettingsDao extends JpaAbstractDao return DaoUtil.pageToPageData(tenantRepository.findTenantsIds(DaoUtil.toPageable(pageLink))).mapData(TenantId::fromUUID); } + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + @Override public List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId) { return tenantRepository.findTenantIdsByTenantProfileId(tenantProfileId.getId()).stream() diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index 7b060e0b43..8c123b12fb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -42,5 +42,6 @@ public interface ApiUsageStateRepository extends JpaRepository imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java index 38642a0161..c5a8763209 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java @@ -16,6 +16,9 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; @@ -28,6 +31,8 @@ public interface UserAuthSettingsRepository extends JpaRepository findByTenantId(UUID tenantId, PageLink pageLink) { + return findTenantWidgetsBundlesByTenantId(tenantId, pageLink); + } + + @Override + public WidgetsBundleId getExternalIdByInternal(WidgetsBundleId internalId) { + return Optional.ofNullable(widgetsBundleRepository.getExternalIdById(internalId.getId())) + .map(WidgetsBundleId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.WIDGETS_BUNDLE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 0de8d0220b..188b42f646 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import java.util.UUID; @@ -27,7 +28,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 4/23/2017. */ -public interface WidgetsBundleRepository extends JpaRepository { +public interface WidgetsBundleRepository extends JpaRepository, ExportableEntityRepository { WidgetsBundleEntity findWidgetsBundleByTenantIdAndAlias(UUID tenantId, String alias); @@ -49,4 +50,10 @@ public interface WidgetsBundleRepository extends JpaRepository { + + public TenantCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.TENANTS_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantEvictEvent.java new file mode 100644 index 0000000000..2bd28ae511 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantEvictEvent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class TenantEvictEvent { + private final TenantId tenantId; + // for exists tenant cache + private final boolean isExistsTenant; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsCaffeineCache.java new file mode 100644 index 0000000000..c7b25a5059 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("TenantExistsCache") +public class TenantExistsCaffeineCache extends CaffeineTbTransactionalCache { + + public TenantExistsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.TENANTS_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java new file mode 100644 index 0000000000..4f938bf690 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("TenantExistsCache") +public class TenantExistsRedisCache extends RedisTbTransactionalCache { + + public TenantExistsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java new file mode 100644 index 0000000000..195e781755 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("TenantCache") +public class TenantRedisCache extends RedisTbTransactionalCache { + + public TenantRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 0590286714..35a22f8281 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -18,7 +18,11 @@ package org.thingsboard.server.dao.tenant; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; @@ -31,8 +35,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.resource.ResourceService; @@ -41,6 +44,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -51,7 +55,7 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Service @Slf4j -public class TenantServiceImpl extends AbstractEntityService implements TenantService { +public class TenantServiceImpl extends AbstractCachedEntityService implements TenantService { private static final String DEFAULT_TENANT_REGION = "Global"; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -63,6 +67,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private TenantProfileService tenantProfileService; @Autowired + @Lazy private UserService userService; @Autowired @@ -77,12 +82,10 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private DeviceProfileService deviceProfileService; + @Lazy @Autowired private ApiUsageStateService apiUsageStateService; - @Autowired - private EntityViewService entityViewService; - @Autowired private WidgetsBundleService widgetsBundleService; @@ -107,11 +110,29 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private QueueService queueService; + @Autowired + private AdminSettingsService adminSettingsService; + + @Autowired + protected TbTransactionalCache existsTenantCache; + + @TransactionalEventListener(classes = TenantEvictEvent.class) + @Override + public void handleEvictEvent(TenantEvictEvent event) { + TenantId tenantId = event.getTenantId(); + cache.evict(TenantCacheKey.fromId(tenantId)); + if (event.isExistsTenant()) { + existsTenantCache.evict(TenantCacheKey.fromIdExists(tenantId)); + } + } + @Override public Tenant findTenantById(TenantId tenantId) { log.trace("Executing findTenantById [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - return tenantDao.findById(tenantId, tenantId.getId()); + + return cache.getAndPutInTransaction(TenantCacheKey.fromId(tenantId), + () -> tenantDao.findById(tenantId, tenantId.getId()), true); } @Override @@ -123,12 +144,13 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public ListenableFuture findTenantByIdAsync(TenantId callerId, TenantId tenantId) { - log.trace("Executing TenantIdAsync [{}]", tenantId); + log.trace("Executing findTenantByIdAsync [{}]", tenantId); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); return tenantDao.findByIdAsync(callerId, tenantId.getId()); } @Override + @Transactional public Tenant saveTenant(Tenant tenant) { log.trace("Executing saveTenant [{}]", tenant); tenant.setRegion(DEFAULT_TENANT_REGION); @@ -138,6 +160,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } tenantValidator.validate(tenant, Tenant::getId); Tenant savedTenant = tenantDao.save(tenant.getId(), tenant); + publishEvictEvent(new TenantEvictEvent(savedTenant.getId(), false)); if (tenant.getId() == null) { deviceProfileService.createDefaultDeviceProfile(savedTenant.getId()); apiUsageStateService.createDefaultApiUsageState(savedTenant.getId(), null); @@ -146,6 +169,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } @Override + @Transactional(timeout = 60 * 60) public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); @@ -164,7 +188,9 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe otaPackageService.deleteOtaPackagesByTenantId(tenantId); rpcService.deleteAllRpcByTenantId(tenantId); queueService.deleteQueuesByTenantId(tenantId); + adminSettingsService.deleteAdminSettingsByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); + publishEvictEvent(new TenantEvictEvent(tenantId, true)); deleteEntityRelations(tenantId, tenantId); } @@ -194,17 +220,29 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe tenantsRemover.removeEntities(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); } - private PaginatedRemover tenantsRemover = - new PaginatedRemover<>() { + @Override + public PageData findTenantsIds(PageLink pageLink) { + log.trace("Executing findTenantsIds"); + Validator.validatePageLink(pageLink); + return tenantDao.findTenantsIds(pageLink); + } + + @Override + public boolean tenantExists(TenantId tenantId) { + return existsTenantCache.getAndPutInTransaction(TenantCacheKey.fromIdExists(tenantId), + () -> tenantDao.existsById(tenantId, tenantId.getId()), false); + } - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return tenantDao.findTenants(tenantId, pageLink); - } + private PaginatedRemover tenantsRemover = new PaginatedRemover<>() { - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(TenantId.fromUUID(entity.getUuidId())); - } - }; + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return tenantDao.findTenants(tenantId, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(TenantId.fromUUID(entity.getUuidId())); + } + }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index db52629d8d..f2bd8e47c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.dao.tenant.TenantProfileDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.ArrayList; @@ -52,16 +52,16 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A private final ApiUsageStateDao apiUsageStateDao; private final TenantProfileDao tenantProfileDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final TimeseriesService tsService; private final DataValidator apiUsageStateValidator; public ApiUsageStateServiceImpl(ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, - TenantDao tenantDao, @Lazy TimeseriesService tsService, + TenantService tenantService, @Lazy TimeseriesService tsService, DataValidator apiUsageStateValidator) { this.apiUsageStateDao = apiUsageStateDao; this.tenantProfileDao = tenantProfileDao; - this.tenantDao = tenantDao; + this.tenantService = tenantService; this.tsService = tsService; this.apiUsageStateValidator = apiUsageStateValidator; } @@ -118,7 +118,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A if (entityId.getEntityType() == EntityType.TENANT && !entityId.equals(TenantId.SYS_TENANT_ID)) { tenantId = (TenantId) entityId; - Tenant tenant = tenantDao.findById(tenantId, tenantId.getId()); + Tenant tenant = tenantService.findTenantById(tenantId); TenantProfile tenantProfile = tenantProfileDao.findById(tenantId, tenant.getTenantProfileId().getId()); TenantProfileConfiguration configuration = tenantProfile.getProfileData().getConfiguration(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 24b5dc837e..7c8a8ca27b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Value; @@ -50,6 +51,7 @@ import static org.thingsboard.server.dao.service.Validator.validateString; @Service @Slf4j +@RequiredArgsConstructor public class UserServiceImpl extends AbstractEntityService implements UserService { public static final String USER_PASSWORD_HISTORY = "userPasswordHistory"; @@ -73,20 +75,6 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; - public UserServiceImpl(UserDao userDao, - UserCredentialsDao userCredentialsDao, - UserAuthSettingsDao userAuthSettingsDao, - DataValidator userValidator, - DataValidator userCredentialsValidator, - ApplicationEventPublisher eventPublisher) { - this.userDao = userDao; - this.userCredentialsDao = userCredentialsDao; - this.userAuthSettingsDao = userAuthSettingsDao; - this.userValidator = userValidator; - this.userCredentialsValidator = userCredentialsValidator; - this.eventPublisher = eventPublisher; - } - @Override public User findUserByEmail(TenantId tenantId, String email) { log.trace("Executing findUserByEmail [{}]", email); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleDao.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleDao.java index 709ecbb58b..4658d78980 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleDao.java @@ -16,17 +16,19 @@ package org.thingsboard.server.dao.widget; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.UUID; /** * The Interface WidgetsBundleDao. */ -public interface WidgetsBundleDao extends Dao { +public interface WidgetsBundleDao extends Dao, ExportableEntityDao { /** * Save or update widgets bundle object diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index b74d935c12..d4537a700d 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -51,3 +51,23 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); + +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_node_external_id ON rule_node(rule_chain_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_node_type ON rule_node(type); \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index ad9835af33..953276cbee 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -34,6 +34,7 @@ call insert_tb_schema_settings(); CREATE TABLE IF NOT EXISTS admin_settings ( id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, + tenant_id uuid NOT NULL, created_time bigint NOT NULL, json_value varchar, key varchar(255) @@ -82,6 +83,7 @@ CREATE TABLE IF NOT EXISTS asset ( search_text varchar(255), tenant_id uuid, type varchar(255), + external_id uuid, CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) ); @@ -141,7 +143,8 @@ CREATE TABLE IF NOT EXISTS customer ( state varchar(255), tenant_id uuid, title varchar(255), - zip varchar(255) + zip varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS dashboard ( @@ -154,7 +157,8 @@ CREATE TABLE IF NOT EXISTS dashboard ( title varchar(255), mobile_hide boolean DEFAULT false, mobile_order int, - image varchar(1000000) + image varchar(1000000), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_chain ( @@ -168,7 +172,8 @@ CREATE TABLE IF NOT EXISTS rule_chain ( root boolean, debug_mode boolean, search_text varchar(255), - tenant_id uuid + tenant_id uuid, + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node ( @@ -180,7 +185,8 @@ CREATE TABLE IF NOT EXISTS rule_node ( type varchar(255), name varchar(255), debug_mode boolean, - search_text varchar(255) + search_text varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node_state ( @@ -249,6 +255,7 @@ CREATE TABLE IF NOT EXISTS device_profile ( default_dashboard_id uuid, default_queue_id uuid, provision_device_key varchar, + external_id uuid, CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), @@ -293,6 +300,7 @@ CREATE TABLE IF NOT EXISTS device ( tenant_id uuid, firmware_id uuid, software_id uuid, + external_id uuid, CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), @@ -416,7 +424,8 @@ CREATE TABLE IF NOT EXISTS widgets_bundle ( tenant_id uuid, title varchar(255), image varchar(1000000), - description varchar(255) + description varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS entity_view ( @@ -432,7 +441,8 @@ CREATE TABLE IF NOT EXISTS entity_view ( start_ts bigint, end_ts bigint, search_text varchar(255), - additional_info varchar + additional_info varchar, + external_id uuid ); CREATE TABLE IF NOT EXISTS ts_kv_latest diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index ef163b2a06..964e6229f1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -65,6 +65,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; +import org.thingsboard.server.dao.rpc.RpcService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; @@ -170,6 +171,9 @@ public abstract class AbstractServiceTest { @Autowired protected OtaPackageService otaPackageService; + @Autowired + protected RpcService rpcService; + @Autowired protected QueueService queueService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 6f088c79f6..ade4ce8c28 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -97,26 +97,26 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(0, otaPackageService.sumDataSizeByTenantId(tenantId)); - createFirmware(tenantId, "1"); + createAndSaveFirmware(tenantId, "1"); Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); thrown.expect(DataValidationException.class); thrown.expectMessage(String.format("Failed to create the ota package, files size limit is exhausted %d bytes!", DATA_SIZE)); - createFirmware(tenantId, "2"); + createAndSaveFirmware(tenantId, "2"); } @Test public void sumDataSizeByTenantId() { Assert.assertEquals(0, otaPackageService.sumDataSizeByTenantId(tenantId)); - createFirmware(tenantId, "0.1"); + createAndSaveFirmware(tenantId, "0.1"); Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; List packages = new ArrayList<>(maxSumDataSize); for (int i = 2; i <= maxSumDataSize; i++) { - packages.add(createFirmware(tenantId, "0." + i)); + packages.add(createAndSaveFirmware(tenantId, "0." + i)); Assert.assertEquals(i, otaPackageService.sumDataSizeByTenantId(tenantId)); } @@ -419,15 +419,15 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testSaveFirmwareWithExistingTitleAndVersion() { - createFirmware(tenantId, VERSION); + createAndSaveFirmware(tenantId, VERSION); thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage with such title and version already exists!"); - createFirmware(tenantId, VERSION); + createAndSaveFirmware(tenantId, VERSION); } @Test public void testDeleteFirmwareWithReferenceByDevice() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); Device device = new Device(); device.setTenantId(tenantId); @@ -448,7 +448,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testUpdateDeviceProfileId() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); try { thrown.expect(DataValidationException.class); @@ -493,7 +493,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testFindFirmwareById() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertNotNull(foundFirmware); @@ -519,7 +519,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testDeleteFirmware() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertNotNull(foundFirmware); @@ -532,7 +532,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public void testFindTenantFirmwaresByTenantId() { List firmwares = new ArrayList<>(); for (int i = 0; i < 165; i++) { - OtaPackageInfo info = new OtaPackageInfo(createFirmware(tenantId, VERSION + i)); + OtaPackageInfo info = new OtaPackageInfo(createAndSaveFirmware(tenantId, VERSION + i)); info.setHasData(true); firmwares.add(info); } @@ -579,7 +579,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public void testFindTenantFirmwaresByTenantIdAndHasData() { List firmwares = new ArrayList<>(); for (int i = 0; i < 165; i++) { - firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createFirmware(tenantId, VERSION + i)))); + firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i)))); } OtaPackageInfo firmwareWithUrl = new OtaPackageInfo(); @@ -695,7 +695,15 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { otaPackageService.saveOtaPackageInfo(firmwareInfo, true); } - private OtaPackage createFirmware(TenantId tenantId, String version) { + private OtaPackage createAndSaveFirmware(TenantId tenantId, String version) { + return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId)); + } + + public static OtaPackage createFirmware( + TenantId tenantId, + String version, + DeviceProfileId deviceProfileId + ) { OtaPackage firmware = new OtaPackage(); firmware.setTenantId(tenantId); firmware.setDeviceProfileId(deviceProfileId); @@ -708,6 +716,6 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setChecksum(CHECKSUM); firmware.setData(DATA); firmware.setDataSize(DATA_SIZE); - return otaPackageService.saveOtaPackage(firmware); + return firmware; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java index 1f167976e5..75ac95d470 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java @@ -57,13 +57,13 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertTrue(saveRelation(relation)); - Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON)); } @Test @@ -80,9 +80,9 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertTrue(relationService.deleteRelationAsync(SYSTEM_TENANT_ID, relationA).get()); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); Assert.assertTrue(relationService.deleteRelationAsync(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); } @@ -116,11 +116,11 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { saveRelation(relationA); saveRelation(relationB); - Assert.assertNull(relationService.deleteEntityRelationsAsync(SYSTEM_TENANT_ID, childId).get()); + relationService.deleteEntityRelations(SYSTEM_TENANT_ID, childId); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); } @Test diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 6017883514..d893a1ba40 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -17,26 +17,77 @@ package org.thingsboard.server.dao.service; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbCacheValueWrapper; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rpc.Rpc; +import org.thingsboard.server.common.data.rpc.RpcStatus; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.tenant.TenantCacheKey; +import org.thingsboard.server.dao.tenant.TenantDao; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; +import java.util.Objects; +import java.util.Set; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; public abstract class BaseTenantServiceTest extends AbstractServiceTest { - + private IdComparator idComparator = new IdComparator<>(); + @SpyBean + protected TenantDao tenantDao; + + @Autowired + protected TbTransactionalCache cache; + + @Autowired + protected TbTransactionalCache existsTenantCache; + @Test public void testSaveTenant() { Tenant tenant = new Tenant(); @@ -103,7 +154,6 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { @Test public void testFindTenants() { - List tenants = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = tenantService.findTenants(pageLink); @@ -275,4 +325,382 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTenantProfileId(isolatedTenantProfile.getId()); tenantService.saveTenant(tenant); } + + @Test + public void testGettingTenantAddingItToCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Mockito.reset(tenantDao); + + verify(tenantDao, Mockito.times(0)).findById(any(), any()); + tenantService.findTenantById(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + var cachedTenant = cache.get(TenantCacheKey.fromId(savedTenant.getId())); + Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertEquals(savedTenant, cachedTenant.get()); + + for (int i = 0; i < 100; i++) { + tenantService.findTenantById(savedTenant.getId()); + } + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testExistsTenantAddingResultToCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Mockito.reset(tenantDao); + //fromIdExists invoked from device profile validator + existsTenantCache.evict(TenantCacheKey.fromIdExists(savedTenant.getTenantId())); + + verify(tenantDao, Mockito.times(0)).existsById(any(), any()); + tenantService.tenantExists(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + var isExists = existsTenantCache.get(TenantCacheKey.fromIdExists(savedTenant.getId())); + Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", isExists); + + for (int i = 0; i < 100; i++) { + tenantService.tenantExists(savedTenant.getId()); + } + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdatingExistingTenantEvictCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + tenantService.findTenantById(savedTenant.getId()); + + var cachedTenant = cache.get(TenantCacheKey.fromId(savedTenant.getId())); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertEquals(savedTenant, cachedTenant.get()); + + savedTenant.setTitle("My new tenant"); + savedTenant = tenantService.saveTenant(savedTenant); + + Mockito.reset(tenantDao); + + cachedTenant = cache.get(TenantCacheKey.fromId(savedTenant.getId())); + Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); + + verify(tenantDao, Mockito.times(0)).findById(any(), any()); + tenantService.findTenantById(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testRemovingTenantEvictCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + tenantService.findTenantById(savedTenant.getId()); + tenantService.tenantExists(savedTenant.getId()); + + var cachedTenant = + cache.get(TenantCacheKey.fromId(savedTenant.getId())); + var cachedExists = + existsTenantCache.get(TenantCacheKey.fromIdExists(savedTenant.getId())); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedExists); + + tenantService.deleteTenant(savedTenant.getId()); + cachedTenant = + cache.get(TenantCacheKey.fromId(savedTenant.getId())); + cachedExists = + existsTenantCache.get(TenantCacheKey.fromIdExists(savedTenant.getId())); + + + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedTenant); + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedExists); + } + + @Test + public void testDeleteTenantDeletingAllRelatedEntities() throws Exception { + TenantProfile profile = createAndSaveTenantProfile(); + Tenant tenant = createAndSaveTenant(profile); + User user = createAndSaveUserFor(tenant); + Customer customer = createAndSaveCustomerFor(tenant); + WidgetsBundle widgetsBundle = createAndSaveWidgetBundleFor(tenant); + DeviceProfile deviceProfile = createAndSaveDeviceProfileWithProfileDataFor(tenant); + Device device = createAndSaveDeviceFor(tenant, customer, deviceProfile); + EntityView entityView = createAndSaveEntityViewFor(tenant, customer, device); + Asset asset = createAndSaveAssetFor(tenant, customer); + Dashboard dashboard = createAndSaveDashboardFor(tenant, customer); + RuleChain ruleChain = createAndSaveRuleChainFor(tenant); + Edge edge = createAndSaveEdgeFor(tenant); + OtaPackage otaPackage = createAndSaveOtaPackageFor(tenant, deviceProfile); + TbResource resource = createAndSaveResourceFor(tenant); + Rpc rpc = createAndSaveRpcFor(tenant, device); + + tenantService.deleteTenant(tenant.getId()); + + Assert.assertNull(tenantService.findTenantById(tenant.getId())); + assertCustomerIsDeleted(tenant, customer); + assertWidgetsBundleIsDeleted(tenant, widgetsBundle); + assertEntityViewIsDeleted(tenant, device, entityView); + assertAssetIsDeleted(tenant, asset); + assertDeviceIsDeleted(tenant, device); + assertDeviceProfileIsDeleted(tenant, deviceProfile); + assertDashboardIsDeleted(tenant, dashboard); + assertEdgeIsDeleted(tenant, edge); + assertTenantAdminIsDeleted(tenant); + assertUserIsDeleted(tenant, user); + Assert.assertNull(ruleChainService.findRuleChainById(tenant.getId(), ruleChain.getId())); + Assert.assertNull(apiUsageStateService.findTenantApiUsageState(tenant.getId())); + assertResourceIsDeleted(tenant, resource); + assertOtaPackageIsDeleted(tenant, otaPackage); + Assert.assertNull(rpcService.findById(tenant.getId(), rpc.getId())); + + tenantProfileService.deleteTenantProfile(TenantId.SYS_TENANT_ID, profile.getId()); + } + + private void assertOtaPackageIsDeleted(Tenant tenant, OtaPackage otaPackage) { + assertThat(otaPackageService.findOtaPackageById(tenant.getId(), otaPackage.getId())) + .as("otaPackage").isNull(); + PageLink pageLinkOta = new PageLink(1); + PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(tenant.getId(), pageLinkOta); + Assert.assertEquals(0, pageDataOta.getTotalElements()); + } + + private void assertResourceIsDeleted(Tenant tenant, TbResource resource) { + assertThat(resourceService.findResourceById(tenant.getId(), resource.getId())) + .as("resource").isNull(); + PageLink pageLinkResources = new PageLink(1); + PageData tenantResources = + resourceService.findAllTenantResourcesByTenantId(tenant.getId(), pageLinkResources); + Assert.assertEquals(0, tenantResources.getTotalElements()); + } + + private void assertUserIsDeleted(Tenant tenant, User user) { + assertThat(userService.findUserById(tenant.getId(), user.getId())) + .as("user").isNull(); + PageLink pageLinkUsers = new PageLink(1); + PageData users = + userService.findUsersByTenantId(tenant.getId(), pageLinkUsers); + Assert.assertEquals(0, users.getTotalElements()); + } + + private void assertTenantAdminIsDeleted(Tenant savedTenant) { + PageLink pageLinkTenantAdmins = new PageLink(1); + PageData tenantAdmins = + userService.findTenantAdmins(savedTenant.getId(), pageLinkTenantAdmins); + Assert.assertEquals(0, tenantAdmins.getTotalElements()); + } + + private void assertEdgeIsDeleted(Tenant tenant, Edge edge) { + assertThat(edgeService.findEdgeById(tenant.getId(), edge.getId())) + .as("edge").isNull(); + PageLink pageLinkEdges = new PageLink(1); + PageData edges = edgeService.findEdgesByTenantId(tenant.getId(), pageLinkEdges); + Assert.assertEquals(0, edges.getTotalElements()); + } + + private void assertDashboardIsDeleted(Tenant tenant, Dashboard dashboard) { + assertThat(dashboardService.findDashboardById(tenant.getId(), dashboard.getId())) + .as("dashboard").isNull(); + PageLink pageLinkDashboards = new PageLink(1); + PageData dashboards = + dashboardService.findDashboardsByTenantId(tenant.getId(), pageLinkDashboards); + Assert.assertEquals(0, dashboards.getTotalElements()); + } + + private void assertDeviceProfileIsDeleted(Tenant tenant, DeviceProfile deviceProfile) { + assertThat(deviceProfileService.findDeviceProfileById(tenant.getId(), deviceProfile.getId())) + .as("deviceProfile").isNull(); + PageLink pageLinkDeviceProfiles = new PageLink(1); + PageData profiles = + deviceProfileService.findDeviceProfiles(tenant.getId(), pageLinkDeviceProfiles); + Assert.assertEquals(0, profiles.getTotalElements()); + } + + private void assertDeviceIsDeleted(Tenant tenant, Device device) { + assertThat(deviceService.findDeviceById(tenant.getId(), device.getId())) + .as("device").isNull(); + PageLink pageLinkDevices = new PageLink(1); + PageData devices = + deviceService.findDevicesByTenantId(tenant.getId(), pageLinkDevices); + Assert.assertEquals(0, devices.getTotalElements()); + } + + private void assertAssetIsDeleted(Tenant tenant, Asset asset) { + assertThat(assetService.findAssetById(tenant.getId(), asset.getId())) + .as("asset").isNull(); + PageLink pageLinkAssets = new PageLink(1); + PageData assets = + assetService.findAssetsByTenantId(tenant.getId(), pageLinkAssets); + Assert.assertEquals(0, assets.getTotalElements()); + } + + private void assertEntityViewIsDeleted(Tenant tenant, Device device, EntityView entityView) { + assertThat(entityViewService.findEntityViewById(tenant.getId(), entityView.getId())) + .as("entityView").isNull(); + List entityViews = + entityViewService.findEntityViewsByTenantIdAndEntityId(tenant.getId(), device.getId()); + Assert.assertTrue(entityViews.isEmpty()); + } + + private void assertWidgetsBundleIsDeleted(Tenant tenant, WidgetsBundle widgetsBundle) { + assertThat(widgetsBundleService.findWidgetsBundleById(tenant.getId(), widgetsBundle.getId())) + .as("widgetBundle").isNull(); + List widgetsBundlesByTenantId = + widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(tenant.getId()); + Assert.assertTrue(widgetsBundlesByTenantId.isEmpty()); + } + + private void assertCustomerIsDeleted(Tenant tenant, Customer customer) { + assertThat(customerService.findCustomerById(tenant.getId(), customer.getId())) + .as("customer").isNull(); + PageLink pageLinkCustomer = new PageLink(1); + PageData pageDataCustomer = customerService + .findCustomersByTenantId(tenant.getId(), pageLinkCustomer); + Assert.assertEquals(0, pageDataCustomer.getTotalElements()); + } + + private Rpc createAndSaveRpcFor(Tenant tenant, Device device) { + Rpc rpc = new Rpc(); + rpc.setTenantId(tenant.getId()); + rpc.setDeviceId(device.getId()); + rpc.setStatus(RpcStatus.QUEUED); + rpc.setRequest(JacksonUtil.toJsonNode("{}")); + return rpcService.save(rpc); + } + + private TbResource createAndSaveResourceFor(Tenant tenant) { + TbResource resource = new TbResource(); + resource.setTenantId(tenant.getId()); + resource.setTitle("Test resource"); + resource.setResourceType(ResourceType.LWM2M_MODEL); + resource.setFileName("filename.txt"); + resource.setResourceKey("Test resource key"); + resource.setData("Some super test data"); + return resourceService.saveResource(resource); + } + + private OtaPackage createAndSaveOtaPackageFor(Tenant tenant, DeviceProfile deviceProfile) { + return otaPackageService.saveOtaPackage( + BaseOtaPackageServiceTest.createFirmware( + tenant.getId(), "2", deviceProfile.getId()) + ); + } + + private Edge createAndSaveEdgeFor(Tenant tenant) { + Edge edge = constructEdge(tenant.getId(), "Test edge", "Simple"); + return edgeService.saveEdge(edge); + } + + private RuleChain createAndSaveRuleChainFor(Tenant tenant) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenant.getId()); + ruleChain.setName("Test rule chain"); + ruleChain.setType(RuleChainType.CORE); + return ruleChainService.saveRuleChain(ruleChain); + } + + private Dashboard createAndSaveDashboardFor(Tenant tenant, Customer customer) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenant.getId()); + dashboard.setTitle("Test dashboard"); + dashboard.setAssignedCustomers(Set.of(customer.toShortCustomerInfo())); + return dashboardService.saveDashboard(dashboard); + } + + private Asset createAndSaveAssetFor(Tenant tenant, Customer customer) { + Asset asset = new Asset(); + asset.setTenantId(tenant.getId()); + asset.setCustomerId(customer.getId()); + asset.setType("Test asset type"); + asset.setName("Test asset type"); + asset.setLabel("Test asset type"); + return assetService.saveAsset(asset); + } + + private EntityView createAndSaveEntityViewFor(Tenant tenant, Customer customer, Device device) { + EntityView entityView = new EntityView(); + entityView.setEntityId(device.getId()); + entityView.setTenantId(tenant.getId()); + entityView.setCustomerId(customer.getId()); + entityView.setType("Test type"); + entityView.setName("Test entity view"); + entityView.setStartTimeMs(0); + entityView.setEndTimeMs(840000); + return entityViewService.saveEntityView(entityView); + } + + private Device createAndSaveDeviceFor(Tenant tenant, Customer customer, DeviceProfile deviceProfile) { + Device device = new Device(); + device.setCustomerId(customer.getId()); + device.setTenantId(tenant.getId()); + device.setType("Test type"); + device.setName("TestType"); + device.setLabel("Test type"); + device.setDeviceProfileId(deviceProfile.getId()); + return deviceService.saveDevice(device); + } + + private DeviceProfile createAndSaveDeviceProfileWithProfileDataFor(Tenant tenant) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenant.getId()); + deviceProfile.setTransportType(DeviceTransportType.MQTT); + deviceProfile.setName("Test device profile"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setTransportConfiguration(new MqttDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + private WidgetsBundle createAndSaveWidgetBundleFor(Tenant tenant) { + WidgetsBundle widgetsBundle = new WidgetsBundle(); + widgetsBundle.setTenantId(tenant.getId()); + widgetsBundle.setTitle("Test widgets bundle"); + widgetsBundle.setAlias("TestWidgetsBundle"); + widgetsBundle.setDescription("Just a simple widgets bundle"); + return widgetsBundleService.saveWidgetsBundle(widgetsBundle); + } + + private Customer createAndSaveCustomerFor(Tenant tenant) { + Customer customer = new Customer(); + customer.setTitle("Test customer"); + customer.setTenantId(tenant.getId()); + customer.setEmail("testCustomer@test.com"); + return customerService.saveCustomer(customer); + } + + private User createAndSaveUserFor(Tenant tenant) { + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setEmail("tenantAdmin@test.com"); + user.setFirstName("tenantAdmin"); + user.setLastName("tenantAdmin"); + user.setTenantId(tenant.getId()); + return userService.saveUser(user); + } + + private Tenant createAndSaveTenant(TenantProfile tenantProfile) { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(tenantProfile.getId()); + return tenantService.saveTenant(tenant); + } + + private TenantProfile createAndSaveTenantProfile() { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test tenant profile"); + return tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); + } } diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 3ebcf5a7ec..063b1275d1 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -35,6 +35,9 @@ cache.specs.entityViews.maxSize=100000 cache.specs.claimDevices.timeToLiveInMinutes=1440 cache.specs.claimDevices.maxSize=100000 +cache.specs.tenants.timeToLiveInMinutes=1440 +cache.specs.tenants.maxSize=100000 + cache.specs.securitySettings.timeToLiveInMinutes=1440 cache.specs.securitySettings.maxSize=100000 diff --git a/dao/src/test/resources/sql/system-data.sql b/dao/src/test/resources/sql/system-data.sql index 1d0d29aec5..bfa33c2284 100644 --- a/dao/src/test/resources/sql/system-data.sql +++ b/dao/src/test/resources/sql/system-data.sql @@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1 '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); /** System settings **/ -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{ "baseUrl": "http://localhost:8080" }' ); -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{ "mailFrom": "Thingsboard ", "smtpProtocol": "smtp", "smtpHost": "localhost", diff --git a/docker/.env b/docker/.env index 11e44fdde8..af0f537603 100644 --- a/docker/.env +++ b/docker/.env @@ -10,6 +10,7 @@ HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport +TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor TB_VERSION=latest diff --git a/docker/.gitignore b/docker/.gitignore index e5313d17cf..9c4c778f28 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -5,5 +5,5 @@ tb-node/db/** tb-node/postgres/** tb-node/cassandra/** tb-transports/*/log -docker/tb-vc-executor/log/** +tb-vc-executor/log/** !.env diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 6983dd3d3e..3d5abd0abe 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -61,3 +61,9 @@ services: tb-snmp-transport: env_file: - queue-confluent.env + tb-vc-executor1: + env_file: + - queue-confluent.env + tb-vc-executor2: + env_file: + - queue-confluent.env diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 2184528e53..09c4554562 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -90,3 +90,13 @@ services: - queue-kafka.env depends_on: - kafka + tb-vc-executor1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-vc-executor2: + env_file: + - queue-kafka.env + depends_on: + - kafka \ No newline at end of file diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 704e9400ae..019e087c48 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -53,6 +53,13 @@ services: tb-snmp-transport: volumes: - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + volumes: postgres-db-volume: @@ -76,3 +83,6 @@ volumes: tb-snmp-transport-log-volume: external: true name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index 0364957ee6..c03132d730 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -23,59 +23,40 @@ services: tb-core1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-pubsub.env + tb-vc-executor2: + env_file: + - queue-pubsub.env + diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index 1eb37709e5..d1acc32014 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-rabbitmq.env + tb-vc-executor2: + env_file: + - queue-rabbitmq.env \ No newline at end of file diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml index c511658b31..6e39de0baa 100644 --- a/docker/docker-compose.service-bus.yml +++ b/docker/docker-compose.service-bus.yml @@ -23,57 +23,39 @@ services: tb-core1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-service-bus.env tb-lwm2m-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-service-bus.env + tb-vc-executor2: + env_file: + - queue-service-bus.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 82fbaeed74..1ba6eda32d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -257,6 +257,38 @@ services: - "8080" env_file: - tb-web-ui.env + tb-vc-executor1: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor1 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 + tb-vc-executor2: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor2 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 5af0f96377..ba945a19df 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -26,3 +26,5 @@ mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log + +mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log diff --git a/docker/tb-vc-executor.env b/docker/tb-vc-executor.env new file mode 100644 index 0000000000..f92e30b78f --- /dev/null +++ b/docker/tb-vc-executor.env @@ -0,0 +1,2 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 diff --git a/docker/tb-vc-executor/conf/logback.xml b/docker/tb-vc-executor/conf/logback.xml new file mode 100644 index 0000000000..dc95b3a885 --- /dev/null +++ b/docker/tb-vc-executor/conf/logback.xml @@ -0,0 +1,51 @@ + + + + + + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.log + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/docker/tb-vc-executor/conf/tb-vc-executor.conf b/docker/tb-vc-executor/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..f140e3fb76 --- /dev/null +++ b/docker/tb-vc-executor/conf/tb-vc-executor.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-vc-executor/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-vc-executor/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-vc-executor.out +export LOADER_PATH=/usr/share/tb-vc-executor/conf diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 417799f64f..de55a3afa3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -34,6 +34,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume"; private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; + private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; private final DockerComposeExecutor dockerCompose; @@ -44,6 +45,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final String tbHttpTransportLogVolume; private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; + private final String tbVcExecutorLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -61,6 +63,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME; tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; + tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -72,6 +75,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); + env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); dockerCompose.withEnv(env); } @@ -104,6 +108,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbSnmpTransportLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); + dockerCompose.invokeDocker(); + dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); @@ -126,10 +133,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/"); copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); + copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); } diff --git a/msa/pom.xml b/msa/pom.xml index 106f2be2a3..dd40348bc8 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -40,6 +40,8 @@ tb + vc-executor + vc-executor-docker js-executor web-ui tb-node diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile new file mode 100644 index 0000000000..af259d8f67 --- /dev/null +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -0,0 +1,32 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk11:bullseye-slim + +COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ + +RUN mkdir -p /home/thingsboard/.config/jgit \ + && chown -R ${pkg.user}:${pkg.user} /home/thingsboard \ + && chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-vc-executor.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-vc-executor.sh"] diff --git a/msa/vc-executor-docker/docker/start-tb-vc-executor.sh b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh new file mode 100755 index 0000000000..4384008fd3 --- /dev/null +++ b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER="/config" +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +export LOADER_PATH=/config,${LOADER_PATH} + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=/config/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml new file mode 100644 index 0000000000..39fc35f32a --- /dev/null +++ b/msa/vc-executor-docker/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor-docker + pom + + ThingsBoard Version Control Executor Microservice + https://thingsboard.io + ThingsBoard Version Control Executor Microservice + + + UTF-8 + ${basedir}/../.. + tb-vc-executor + tb-vc-executor + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard.msa + vc-executor + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-tb-vc-executor-deb + package + + copy + + + + + org.thingsboard.msa + vc-executor + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml new file mode 100644 index 0000000000..0125354371 --- /dev/null +++ b/msa/vc-executor/pom.xml @@ -0,0 +1,140 @@ + + + 4.0.0 + + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor + + ThingsBoard Version Control Executor + https://thingsboard.io + Project for ThingsBoard version control microservice + + + UTF-8 + ${basedir}/../.. + java + false + process-resources + package + tb-vc-executor + false + ${project.build.directory}/windows + ThingsBoard Version Control Executor Service + org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication + + + + + org.thingsboard.common + queue + + + org.thingsboard.common + version-control + + + org.springframework.boot + spring-boot-starter-web + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + + + diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml new file mode 100644 index 0000000000..d62cf2b3f5 --- /dev/null +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -0,0 +1,43 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + diff --git a/msa/vc-executor/src/main/conf/tb-vc-executor.conf b/msa/vc-executor/src/main/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..83287286bb --- /dev/null +++ b/msa/vc-executor/src/main/conf/tb-vc-executor.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java new file mode 100644 index 0000000000..6978f94b51 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -0,0 +1,47 @@ +package org.thingsboard.server.vc; /** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootApplication +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) +public class ThingsboardVersionControlExecutorApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "tb-vc-executor"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardVersionControlExecutorApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java new file mode 100644 index 0000000000..0f4ac69948 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class VersionControlQueueRoutingInfoService implements QueueRoutingInfoService { + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java new file mode 100644 index 0000000000..ebd7ed07b3 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class VersionControlTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + //This dummy implementation is ok since Version Control service does not produce any rule engine messages. + return new TenantRoutingInfo(tenantId, false, false); + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml new file mode 100644 index 0000000000..572d093dde --- /dev/null +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -0,0 +1,35 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml new file mode 100644 index 0000000000..4e430e5ba6 --- /dev/null +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -0,0 +1,188 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# If you enabled process metrics you should also enable 'web-environment'. +spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" +# If you enabled process metrics you should set 'web-application-type' to 'servlet' value. +spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" + +server: + # Server bind address (has no effect if web-environment is disabled). + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port (has no effect if web-environment is disabled). + port: "${HTTP_BIND_PORT:8086}" + +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +queue: + type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + - key: max.poll.records + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + ota: + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +service: + type: "${TB_SERVICE_TYPE:tb-vc-executor}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9273e7bebf..a26b0219ac 100755 --- a/pom.xml +++ b/pom.xml @@ -134,6 +134,7 @@ 1.16.0 1.12 3.0.0 + 6.1.0.202203080745-r 1.0.0 @@ -865,6 +866,11 @@ util ${project.version} + + org.thingsboard.common + version-control + ${project.version} + org.thingsboard.common cache @@ -1881,6 +1887,16 @@ aerogear-otp-java ${aerogear-otp.version} + + org.eclipse.jgit + org.eclipse.jgit + ${jgit.version} + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + ${jgit.version} + diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java index 48f30314c5..d5213da4f2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java @@ -124,7 +124,7 @@ public class TbCreateRelationNode extends TbAbstractRelationActionNode checkRelation(TbContext ctx, SearchDirectionIds sdId, String relationType) { - return ctx.getRelationService().checkRelation(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON); + return ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON); } private ListenableFuture processCreateRelation(TbContext ctx, EntityContainer entityContainer, SearchDirectionIds sdId, String relationType) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java index 361462bc3f..f19a271577 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java @@ -98,7 +98,7 @@ public class TbDeleteRelationNode extends TbAbstractRelationActionNode processSingle(TbContext ctx, TbMsg msg, EntityContainer entityContainer, String relationType) { SearchDirectionIds sdId = processSingleSearchDirection(msg, entityContainer); - return Futures.transformAsync(ctx.getRelationService().checkRelation(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON), + return Futures.transformAsync(ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON), result -> { if (result) { return processSingleDeleteRelation(ctx, sdId, relationType); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 1a26cf97ef..80b319e087 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -105,7 +105,7 @@ public class TbMsgGeneratorNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { log.trace("onMsg, config {}, msg {}", config, msg); if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) { - TbStopWatch sw = TbStopWatch.startNew(); + TbStopWatch sw = TbStopWatch.create(); withCallback(generate(ctx, msg), m -> { log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java index 398c4b77ec..c4bc9641a4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java @@ -82,7 +82,7 @@ public class TbCheckRelationNode implements TbNode { to = EntityIdFactory.getByTypeAndId(config.getEntityType(), config.getEntityId()); from = msg.getOriginator(); } - return ctx.getRelationService().checkRelation(ctx.getTenantId(), from, to, config.getRelationType(), RelationTypeGroup.COMMON); + return ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), from, to, config.getRelationType(), RelationTypeGroup.COMMON); } private ListenableFuture processList(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java index b2a0d8dfc9..eafff21819 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java @@ -113,7 +113,7 @@ public class TbCreateRelationNodeTest { metaData.putValue("type", "AssetType"); msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); @@ -144,7 +144,7 @@ public class TbCreateRelationNodeTest { when(ctx.getRelationService().findByToAndTypeAsync(any(), eq(msg.getOriginator()), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(Collections.singletonList(relation))); when(ctx.getRelationService().deleteRelationAsync(any(), eq(relation))).thenReturn(Futures.immediateFuture(true)); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); @@ -171,7 +171,7 @@ public class TbCreateRelationNodeTest { metaData.putValue("type", "AssetType"); msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index d1ac360d44..8acd638ca4 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -1109,7 +1109,7 @@ public class TbDeviceProfileNodeTest { AttributeKvEntity attributeKvEntityActiveSchedule = new AttributeKvEntity(); attributeKvEntityActiveSchedule.setId(compositeKeyActiveSchedule); attributeKvEntityActiveSchedule.setJsonValue( - "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":8.64e+7,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" + "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" ); attributeKvEntityActiveSchedule.setLastUpdateTs(0L); @@ -1166,6 +1166,8 @@ public class TbDeviceProfileNodeTest { TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); +// Mockito.reset(ctx); + node.onMsg(ctx, msg); verify(ctx).tellSuccess(msg); verify(ctx).enqueueForTellNext(theMsg, "Alarm Created"); diff --git a/ui-ngx/angular.json b/ui-ngx/angular.json index f4d4412723..da4d21a84b 100644 --- a/ui-ngx/angular.json +++ b/ui-ngx/angular.json @@ -79,7 +79,8 @@ "node_modules/leaflet.markercluster/dist/MarkerCluster.Default.css", "node_modules/@geoman-io/leaflet-geoman-free/dist/leaflet-geoman.css", "node_modules/prismjs/themes/prism.css", - "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css" + "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css", + "node_modules/ace-diff/dist/ace-diff.min.css" ], "stylePreprocessorOptions": { "includePaths": [ @@ -130,7 +131,8 @@ "jstree", "qrcode", "wcwidth", - "leaflet-polylinedecorator" + "leaflet-polylinedecorator", + "ace-diff" ] }, "configurations": { diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 53b7fc6190..c9db64ddc2 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -41,6 +41,7 @@ "@ngx-translate/core": "^13.0.0", "@ngx-translate/http-loader": "^6.0.0", "ace-builds": "^1.4.13", + "ace-diff": "^3.0.3", "angular-gridster2": "~12.1.1", "angular2-hotkeys": "^2.4.0", "canvas-gauges": "^2.1.7", @@ -106,6 +107,7 @@ "@angular/compiler-cli": "^12.2.13", "@angular/language-service": "^12.2.13", "@ngtools/webpack": "~12.2.13", + "@types/ace-diff": "^2.1.1", "@types/canvas-gauges": "^2.1.4", "@types/flot": "^0.0.32", "@types/jasmine": "~3.10.2", diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 6edbdcd5a2..a60720e61d 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -23,7 +23,8 @@ export enum AuthActionTypes { UNAUTHENTICATED = '[Auth] Unauthenticated', LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', - UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id' + UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', + UPDATE_HAS_REPOSITORY = '[Auth] Change Has Repository' } export class ActionAuthAuthenticated implements Action { @@ -54,5 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } +export class ActionAuthUpdateHasRepository implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_REPOSITORY; + + constructor(readonly payload: { hasRepository: boolean }) {} +} + export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasRepository; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 9267d48e28..33b84945aa 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,6 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; + hasRepository: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index be2712300d..cace62660e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -23,7 +23,8 @@ const emptyUserAuthState: AuthPayload = { userTokenAccessEnabled: false, forceFullscreen: false, allowedDashboardIds: [], - edgesSupportEnabled: false + edgesSupportEnabled: false, + hasRepository: false }; export const initialState: AuthState = { @@ -54,6 +55,9 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; + case AuthActionTypes.UPDATE_HAS_REPOSITORY: + return { ...state, ...action.payload}; + default: return state; } diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index aaadf00ec3..4a63dbcbed 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,6 +55,11 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); +export const selectHasRepository = createSelector( + selectAuthState, + (state: AuthState) => state.hasRepository +); + export function getCurrentAuthState(store: Store): AuthState { let state: AuthState; store.pipe(select(selectAuth), take(1)).subscribe( diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index e26d7813da..c7b1a283d3 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -468,17 +468,27 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } + private loadHasRepository(authUser: AuthUser): Observable { + if (authUser.authority === Authority.TENANT_ADMIN) { + return this.http.get('/api/admin/repositorySettings/exists', defaultHttpOptions()); + } else { + return of(false); + } + } + private loadSystemParams(authPayload: AuthPayload): Observable { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), + this.loadHasRepository(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled}; + const hasRepository: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasRepository}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 8a12b998da..9b8c3b76b9 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -15,16 +15,21 @@ /// import { Injectable } from '@angular/core'; -import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable } from 'rxjs'; +import { defaultHttpOptions, defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, + RepositorySettings, MailServerSettings, SecuritySettings, TestSmsRequest, - UpdateMessage + UpdateMessage, AutoCommitSettings } from '@shared/models/settings.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { tap } from 'rxjs/operators'; +import { AuthUser } from '@shared/models/user.model'; +import { Authority } from '@shared/models/authority.enum'; @Injectable({ providedIn: 'root' @@ -32,7 +37,8 @@ import { export class AdminService { constructor( - private http: HttpClient + private http: HttpClient, + private entitiesVersionControlService: EntitiesVersionControlService ) { } public getAdminSettings(key: string, config?: RequestConfig): Observable> { @@ -64,6 +70,50 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } + public getRepositorySettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveRepositorySettings(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings', repositorySettings, + defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); + } + + public deleteRepositorySettings(config?: RequestConfig) { + return this.http.delete('/api/admin/repositorySettings', defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); + } + + public checkRepositoryAccess(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings/checkAccess', repositorySettings, defaultHttpOptionsFromConfig(config)); + } + + public getAutoCommitSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); + } + + public autoCommitSettingsExists(config?: RequestConfig): Observable { + return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); + } + + public saveAutoCommitSettings(autoCommitSettings: AutoCommitSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/autoCommitSettings', autoCommitSettings, defaultHttpOptionsFromConfig(config)); + } + + public deleteAutoCommitSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/autoCommitSettings', defaultHttpOptionsFromConfig(config)); + } + public checkUpdates(config?: RequestConfig): Observable { return this.http.get(`/api/admin/updates`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts new file mode 100644 index 0000000000..bb4a10e4e5 --- /dev/null +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -0,0 +1,147 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable, of } from 'rxjs'; +import { + BranchInfo, EntityDataDiff, EntityDataInfo, EntityLoadError, entityLoadErrorTranslationMap, EntityLoadErrorType, + EntityVersion, + VersionCreateRequest, + VersionCreationResult, + VersionLoadRequest, VersionLoadResult +} from '@shared/models/vc.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, tap } from 'rxjs/operators'; +import { TranslateService } from '@ngx-translate/core'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Injectable({ + providedIn: 'root' +}) +export class EntitiesVersionControlService { + + branchList: Array = null; + + constructor( + private http: HttpClient, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private store: Store + ) { + + this.store.pipe(select(selectIsUserLoaded)).subscribe( + () => { + this.branchList = null; + } + ); + } + + public clearBranchList(): void { + this.branchList = null; + } + + public listBranches(): Observable> { + if (!this.branchList) { + return this.http.get>('/api/entities/vc/branches', + defaultHttpOptionsFromConfig({ignoreErrors: true, ignoreLoading: false})).pipe( + catchError(() => of([] as Array)), + tap((list) => { + this.branchList = list; + }) + ); + } else { + return of(this.branchList); + } + } + + public getEntityDataInfo(externalEntityId: EntityId, + versionId: string, + config?: RequestConfig): Observable { + return this.http.get(`/api/entities/vc/info/${versionId}/${externalEntityId.entityType}/${externalEntityId.id}`, + defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { + return this.http.post('/api/entities/vc/version', request, defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + const branch = request.branch; + if (this.branchList && !this.branchList.find(b => b.name === branch)) { + this.branchList = null; + } + }) + ); + } + + public listEntityVersions(pageLink: PageLink, branch: string, + externalEntityId: EntityId, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listEntityTypeVersions(pageLink: PageLink, branch: string, + entityType: EntityType, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public listVersions(pageLink: PageLink, branch: string, + config?: RequestConfig): Observable> { + return this.http.get>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, + defaultHttpOptionsFromConfig(config)); + } + + public loadEntitiesVersion(request: VersionLoadRequest, config?: RequestConfig): Observable { + return this.http.post('/api/entities/vc/entity', request, defaultHttpOptionsFromConfig(config)); + } + + public compareEntityDataToVersion(branch: string, + entityId: EntityId, + versionId: string, + config?: RequestConfig): Observable { + return this.http.get(`/api/entities/vc/diff/${branch}/${entityId.entityType}/${entityId.id}?versionId=${versionId}`, + defaultHttpOptionsFromConfig(config)); + } + + public entityLoadErrorToMessage(entityLoadError: EntityLoadError): SafeHtml { + const type = entityLoadError.type; + const messageId = entityLoadErrorTranslationMap.get(type); + const messageArgs = {} as any; + switch (type) { + case EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT: + messageArgs.entityId = entityLoadError.source.id; + break; + case EntityLoadErrorType.MISSING_REFERENCED_ENTITY: + messageArgs.sourceEntityTypeName = + (this.translate.instant(entityTypeTranslations.get(entityLoadError.source.entityType).type) as string).toLowerCase(); + messageArgs.sourceEntityId = entityLoadError.source.id; + messageArgs.targetEntityTypeName = + (this.translate.instant(entityTypeTranslations.get(entityLoadError.target.entityType).type) as string).toLowerCase(); + messageArgs.targetEntityId = entityLoadError.target.id; + break; + } + return this.sanitizer.bypassSecurityTrustHtml(this.translate.instant(messageId, messageArgs)); + } +} diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 1d20b631ae..b1c395c2ba 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -67,7 +67,6 @@ import { AlarmData, AlarmDataQuery, createDefaultEntityDataPageLink, - defaultEntityDataPageLink, EntityData, EntityDataQuery, entityDataToEntityInfo, @@ -233,6 +232,11 @@ export class EntityService { case EntityType.ALARM: console.error('Get Alarm Entity is not implemented!'); break; + case EntityType.DEVICE_PROFILE: + observable = this.getEntitiesByIdsObservable( + (id) => this.deviceProfileService.getDeviceProfileInfo(id, config), + entityIds); + break; } return observable; } @@ -374,6 +378,10 @@ export class EntityService { pageLink.sortOrder.property = 'title'; entitiesObservable = this.otaPackageService.getOtaPackages(pageLink, config); break; + case EntityType.DEVICE_PROFILE: + pageLink.sortOrder.property = 'name'; + entitiesObservable = this.deviceProfileService.getDeviceProfileInfos(pageLink, null, config); + break; } return entitiesObservable; } diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 78c791ba10..ffc4884b7f 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -357,6 +357,13 @@ export class MenuService { path: '/dashboards', icon: 'dashboards' }, + { + id: guid(), + name: 'version-control.version-control', + type: 'link', + path: '/vc', + icon: 'history' + }, { id: guid(), name: 'audit-log.audit-logs', @@ -376,7 +383,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '80px', + height: '160px', icon: 'settings', pages: [ { @@ -392,6 +399,20 @@ export class MenuService { type: 'link', path: '/settings/resources-library', icon: 'folder' + }, + { + id: guid(), + name: 'admin.repository-settings', + type: 'link', + path: '/settings/repository', + icon: 'manage_history' + }, + { + id: guid(), + name: 'admin.auto-commit-settings', + type: 'link', + path: '/settings/auto-commit', + icon: 'settings_backup_restore' } ] } @@ -499,6 +520,16 @@ export class MenuService { } ] }, + { + name: 'version-control.management', + places: [ + { + name: 'version-control.version-control', + icon: 'history', + path: '/vc' + } + ] + }, { name: 'audit-log.audit', places: [ @@ -526,6 +557,16 @@ export class MenuService { name: 'resource.resources-library', icon: 'folder', path: '/settings/resources-library' + }, + { + name: 'admin.repository-settings', + icon: 'manage_history', + path: '/settings/repository', + }, + { + name: 'admin.auto-commit-settings', + icon: 'settings_backup_restore', + path: '/settings/auto-commit' } ] } diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts index 6ff3f78e7d..9d6812e0f9 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts @@ -285,6 +285,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI this.attributeScopeSelectionReadonly = true; } this.mode = 'default'; + this.textSearchMode = false; this.selectedWidgetsBundleAlias = null; this.attributeScope = this.defaultAttributeScope; this.pageLink.textSearch = null; diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index dd7212cf5b..553971005f 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -81,6 +81,13 @@ (click)="isFullscreen = !isFullscreen"> {{ isFullscreen ? 'fullscreen_exit' : 'fullscreen' }} + + + + +
+ +
+
+ + +
+ + +
+
+
+ + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
+
+
+ + + + +
+ admin.no-auto-commit-entities-prompt +
+
+ + + +
+ + +
+ + + +
+ + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss new file mode 100644 index 0000000000..ea8017b9f1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.auto-commit-settings { + margin: 8px; + .mat-divider { + position: relative; + } + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + tb-branch-autocomplete { + min-width: 200px; + max-width: 200px; + display: block; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts new file mode 100644 index 0000000000..fd100f6247 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -0,0 +1,214 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { AbstractControl, FormArray, FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { AutoCommitSettings, AutoVersionCreateConfig } from '@shared/models/settings.models'; +import { TranslateService } from '@ngx-translate/core'; +import { DialogService } from '@core/services/dialog.service'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-auto-commit-settings', + templateUrl: './auto-commit-settings.component.html', + styleUrls: ['./auto-commit-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class AutoCommitSettingsComponent extends PageComponent implements OnInit { + + autoCommitSettingsForm: FormGroup; + settings: AutoCommitSettings = null; + + entityTypes = EntityType; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private sanitizer: DomSanitizer, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.autoCommitSettingsForm = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.adminService.autoCommitSettingsExists().pipe( + catchError(() => of(false)), + mergeMap((hasAutoCommitSettings) => { + if (hasAutoCommitSettings) { + return this.adminService.getAutoCommitSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(settings), {emitEvent: false}); + }); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.autoCommitSettingsForm.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.autoCommitSettingsForm.get('entityTypes') as FormArray).removeAt(index); + this.autoCommitSettingsForm.markAsDirty(); + } + + public addEnabled(): boolean { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + const config: AutoVersionCreateConfig = { + branch: null, + saveAttributes: true, + saveRelations: false, + saveCredentials: true + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + public removeAll() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + entityTypeText(entityTypeControl: AbstractControl): SafeHtml { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: AutoVersionCreateConfig = entityTypeControl.get('config').value; + let message = entityType ? this.translate.instant(entityTypeTranslations.get(entityType).typePlural) : 'Undefined'; + let branchName; + if (config.branch) { + branchName = config.branch; + } else { + branchName = this.translate.instant('version-control.default'); + } + message += ` (${this.translate.instant('version-control.auto-commit-to-branch', {branch: branchName})})`; + return this.sanitizer.bypassSecurityTrustHtml(message); + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + save(): void { + const value: [{entityType: string, config: AutoVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const settings: AutoCommitSettings = {}; + if (value && value.length) { + value.forEach((val) => { + settings[val.entityType] = val.config; + }); + } + this.adminService.saveAutoCommitSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(savedSettings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-auto-commit-settings-title', ), + this.translate.instant('admin.delete-auto-commit-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteAutoCommitSettings().subscribe( + () => { + this.settings = null; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(this.settings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + }); + } + + private prepareEntityTypesFormArray(settings: AutoCommitSettings | null): FormArray { + const entityTypesControls: Array = []; + if (settings) { + for (const entityType of Object.keys(settings)) { + const config = settings[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: AutoVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + branch: [config.branch, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []] + }) + } + ); + return entityTypeControl; + } + + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html new file mode 100644 index 0000000000..9e15076788 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -0,0 +1,83 @@ + +
+
+ +

{{ 'version-control.create-entities-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + version-control.default-sync-strategy + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + + + + +
+
+
+
+ + +
+
+
+
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts new file mode 100644 index 0000000000..bb97bfaedf --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -0,0 +1,111 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + ComplexVersionCreateRequest, + createDefaultEntityTypesVersionCreate, + SyncStrategy, syncStrategyHintMap, syncStrategyTranslationMap, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-complex-version-create', + templateUrl: './complex-version-create.component.html', + styleUrls: ['./version-control.scss'] +}) +export class ComplexVersionCreateComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + popoverComponent: TbPopoverComponent; + + createVersionFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + syncStrategyHints = syncStrategyHintMap; + + resultMessage: SafeHtml; + + versionCreateResult: VersionCreationResult = null; + + versionCreateBranch: string = null; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private sanitizer: DomSanitizer, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + syncStrategy: [SyncStrategy.MERGE, Validators.required], + entityTypes: [createDefaultEntityTypesVersionCreate(), []], + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionCreateResult, this.versionCreateBranch); + } + } + + export(): void { + const request: ComplexVersionCreateRequest = { + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + syncStrategy: this.createVersionFormGroup.get('syncStrategy').value, + entityTypes: this.createVersionFormGroup.get('entityTypes').value, + type: VersionCreateRequestType.COMPLEX + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified && !result.removed) { + this.resultMessage = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.nothing-to-commit')); + } else { + this.resultMessage = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.version-create-result', + {added: result.added, modified: result.modified, removed: result.removed})); + } + this.versionCreateResult = result; + this.versionCreateBranch = request.branch; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html new file mode 100644 index 0000000000..e52be641ce --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -0,0 +1,66 @@ + +
+
+ +

{{ 'version-control.restore-entities-from-version' | translate: {versionName} }}

+ + + + +
+
+
+ + +
+
+
+
+ + +
+
+
+
+ {{ 'version-control.no-entities-restored' | translate }} +
+
+
{{ entityTypeLoadResultMessage(entityTypeLoadResult) }}
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts new file mode 100644 index 0000000000..a212b7b2ce --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts @@ -0,0 +1,120 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + createDefaultEntityTypesVersionLoad, EntityTypeLoadResult, + EntityTypeVersionLoadRequest, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; +import { SafeHtml } from '@angular/platform-browser'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-complex-version-load', + templateUrl: './complex-version-load.component.html', + styleUrls: ['./version-control.scss'] +}) +export class ComplexVersionLoadComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + onClose: (result: VersionLoadResult | null) => void; + + @Input() + popoverComponent: TbPopoverComponent; + + loadVersionFormGroup: FormGroup; + + versionLoadResult: VersionLoadResult = null; + + entityTypeLoadResults: Array = null; + + errorMessage: SafeHtml; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.loadVersionFormGroup = this.fb.group({ + entityTypes: [createDefaultEntityTypesVersionLoad(), []], + }); + } + + entityTypeLoadResultMessage(result: EntityTypeLoadResult): string { + const entityType = result.entityType; + let message = this.translate.instant(entityTypeTranslations.get(entityType).typePlural) + ': '; + const resultMessages: string[] = []; + if (result.created) { + resultMessages.push(this.translate.instant('version-control.created', {created: result.created})); + } + if (result.updated) { + resultMessages.push(this.translate.instant('version-control.updated', {updated: result.updated})); + } + if (result.deleted) { + resultMessages.push(this.translate.instant('version-control.deleted', {deleted: result.deleted})); + } + message += resultMessages.join(', ') + '.'; + return message; + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionLoadResult); + } + } + + restore(): void { + const request: EntityTypeVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + entityTypes: this.loadVersionFormGroup.get('entityTypes').value, + type: VersionLoadRequestType.ENTITY_TYPE + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + this.versionLoadResult = result; + this.entityTypeLoadResults = (result.result || []).filter(res => res.created || res.updated || res.deleted); + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + } + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html new file mode 100644 index 0000000000..928b7a7668 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -0,0 +1,119 @@ + +
+
+ version-control.entities-to-export +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + version-control.sync-strategy + + + {{ 'version-control.default' | translate }} + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + +
+ + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
+
+
+
+ + {{ 'version-control.all-entities' | translate }} + + + +
+
+ + +
+
+
+ version-control.no-entities-to-export-prompt +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss new file mode 100644 index 0000000000..f90ad2a803 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-create { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts new file mode 100644 index 0000000000..6434329447 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -0,0 +1,253 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + SyncStrategy, + syncStrategyTranslationMap +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-entity-types-version-create', + templateUrl: './entity-types-version-create.component.html', + styleUrls: ['./entity-types-version-create.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + } + ] +}) +export class EntityTypesVersionCreateComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionCreateConfig}; + + private propagateChange = null; + + public entityTypesVersionCreateFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + entityTypes = EntityType; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionCreateFormGroup = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionCreateFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionCreateFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionCreateFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionCreateFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + syncStrategy: [config.syncStrategy === null ? 'default' : config.syncStrategy, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []], + allEntities: [config.allEntities, []], + entityIds: [config.entityIds, [Validators.required]] + }) + } + ); + this.updateEntityTypeValidators(entityTypeControl); + entityTypeControl.get('config').get('allEntities').valueChanges.subscribe(() => { + this.updateEntityTypeValidators(entityTypeControl); + }); + return entityTypeControl; + } + + private updateEntityTypeValidators(entityTypeControl: AbstractControl): void { + const allEntities: boolean = entityTypeControl.get('config').get('allEntities').value; + if (allEntities) { + entityTypeControl.get('config').get('entityIds').disable({emitEvent: false}); + } else { + entityTypeControl.get('config').get('entityIds').enable({emitEvent: false}); + } + entityTypeControl.get('config').get('entityIds').updateValueAndValidity({emitEvent: false}); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionCreateConfig = { + syncStrategy: null, + saveAttributes: true, + saveRelations: true, + saveCredentials: true, + allEntities: true, + entityIds: [] + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: EntityTypeVersionCreateConfig = entityTypeControl.get('config').value; + let count = config?.entityIds?.length; + if (!isDefinedAndNotNull(count)) { + count = 0; + } + if (entityType) { + return this.translate.instant((config?.allEntities ? entityTypeTranslations.get(entityType).typePlural + : entityTypeTranslations.get(entityType).list), { count }); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionCreateConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + if ((modelValue[val.entityType].syncStrategy as any) === 'default') { + modelValue[val.entityType].syncStrategy = null; + } + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html new file mode 100644 index 0000000000..bb3c511e0c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -0,0 +1,105 @@ + +
+
+ version-control.entities-to-restore +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.find-existing-entity-by-name' | translate }} + +
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+ version-control.no-entities-to-restore-prompt +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss new file mode 100644 index 0000000000..b3e840198c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-load { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts new file mode 100644 index 0000000000..ef43899c5b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -0,0 +1,248 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit, Renderer2, ViewContainerRef } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes, VersionCreationResult } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { MatCheckbox } from '@angular/material/checkbox/checkbox'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; + +@Component({ + selector: 'tb-entity-types-version-load', + templateUrl: './entity-types-version-load.component.html', + styleUrls: ['./entity-types-version-load.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + } + ] +}) +export class EntityTypesVersionLoadComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionLoadConfig}; + + private propagateChange = null; + + public entityTypesVersionLoadFormGroup: FormGroup; + + entityTypes = EntityType; + + constructor(protected store: Store, + private translate: TranslateService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private viewContainerRef: ViewContainerRef, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionLoadFormGroup = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.entityTypesVersionLoadFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionLoadFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionLoadFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionLoadFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionLoadFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionLoadConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + loadRelations: [config.loadRelations, []], + loadAttributes: [config.loadAttributes, []], + loadCredentials: [config.loadCredentials, []], + removeOtherEntities: [config.removeOtherEntities, []], + findExistingEntityByName: [config.findExistingEntityByName, []] + }) + } + ); + return entityTypeControl; + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionLoadConfig = { + loadAttributes: true, + loadRelations: true, + loadCredentials: true, + removeOtherEntities: false, + findExistingEntityByName: true + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + if (entityType) { + return this.translate.instant(entityTypeTranslations.get(entityType).typePlural); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + onRemoveOtherEntities(removeOtherEntitiesCheckbox: MatCheckbox, entityTypeControl: AbstractControl, $event: Event) { + const removeOtherEntities: boolean = entityTypeControl.get('config.removeOtherEntities').value; + if (!removeOtherEntities) { + $event.preventDefault(); + $event.stopPropagation(); + const trigger = $('.mat-checkbox-frame', removeOtherEntitiesCheckbox._elementRef.nativeElement)[0]; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const removeOtherEntitiesConfirmPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, RemoveOtherEntitiesConfirmComponent, 'bottom', true, null, + { + onClose: (result: boolean | null) => { + removeOtherEntitiesConfirmPopover.hide(); + if (result) { + entityTypeControl.get('config').get('removeOtherEntities').patchValue(true, {emitEvent: true}); + } + } + }, {}, {}, {}, false); + } + } + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionLoadConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html new file mode 100644 index 0000000000..270d42d328 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -0,0 +1,79 @@ + +
+
+ +

{{ 'version-control.create-entity-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
+
+
+
+ + +
+
+
+
{{ resultMessage }}
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts new file mode 100644 index 0000000000..e3bf44560b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -0,0 +1,118 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { Observable, of } from 'rxjs'; +import { EntityType } from '@shared/models/entity-type.models'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-entity-version-create', + templateUrl: './entity-version-create.component.html', + styleUrls: ['./version-control.scss'] +}) +export class EntityVersionCreateComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + entityId: EntityId; + + @Input() + entityName: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + onBeforeCreateVersion: () => Observable; + + @Input() + popoverComponent: TbPopoverComponent; + + createVersionFormGroup: FormGroup; + + entityTypes = EntityType; + + resultMessage: string; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [this.translate.instant('version-control.default-create-entity-version-name', + {entityName: this.entityName}), [Validators.required]], + saveRelations: [false, []], + saveAttributes: [true, []], + saveCredentials: [true, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null, null); + } + } + + export(): void { + const before = this.onBeforeCreateVersion ? this.onBeforeCreateVersion() : of(null); + before.subscribe(() => { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + config: { + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, + saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.saveEntitiesVersion(request).subscribe((result) => { + if (!result.added && !result.modified) { + this.resultMessage = this.translate.instant('version-control.nothing-to-commit'); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else if (this.onClose) { + this.onClose(result, request.branch); + } + }); + }); + } +} + diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html new file mode 100644 index 0000000000..22bebd716f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html @@ -0,0 +1,71 @@ + +
+ +

{{ 'version-control.diff-entity-with-version' | translate: {versionName} }}

+ + + + + {{ 'version-control.differences' | translate : {count: diffCount} }} + + + + +
+
{{ 'version-control.current' | translate }}
+
+
{{ versionIdContent() }}
+
+ +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss new file mode 100644 index 0000000000..cf0bf939b7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../../../../scss/constants'; + +.entity-version-diff-view { + position: relative; + &:not(.tb-fullscreen) { + &.content-ready { + width: 600px; + @media #{$mat-gt-sm} { + width: 800px; + } + @media #{$mat-gt-md} { + width: 1000px + } + @media #{$mat-gt-xmd} { + width: 1200px + } + } + } + &.tb-fullscreen { + height: 100%; + background: #fff; + .bottom-panel { + margin-bottom: 16px; + } + } + .mat-toolbar { + background: #fff; + .mat-divider-vertical { + height: 40px; + margin-right: 16px; + margin-left: 16px; + } + .diff-count { + font-size: 16px; + } + } + .version-title { + margin-left: 4px; + } + .diff-viewer { + position: relative; + border: 1px solid #c0c0c0; + margin-top: 4px; + margin-bottom: 16px; + .acediff__diffLine { + z-index: 1; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts new file mode 100644 index 0000000000..29955af18e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -0,0 +1,328 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectorRef, + Component, + ElementRef, + EventEmitter, + Input, + OnDestroy, + OnInit, + Output, + Renderer2, + ViewChild, + ViewContainerRef, + ViewEncapsulation +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { getAceDiff } from '@shared/models/ace/ace.models'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { entityExportDataToJsonString, VersionLoadResult } from '@shared/models/vc.models'; +import { Ace } from 'ace-builds'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; + +interface DiffInfo { + leftStartLine: number; + leftEndLine: number; + rightStartLine: number; + rightEndLine: number; +} + +@Component({ + selector: 'tb-entity-version-diff', + templateUrl: './entity-version-diff.component.html', + styleUrls: ['./entity-version-diff.component.scss'], + encapsulation: ViewEncapsulation.None +}) +export class EntityVersionDiffComponent extends PageComponent implements OnInit, OnDestroy { + + @ViewChild('diffViewer', {static: true}) + diffViewerElmRef: ElementRef; + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + entityId: EntityId; + + @Input() + externalEntityId: EntityId; + + @Output() + versionRestored = new EventEmitter(); + + @Input() + onClose: () => void; + + @Input() + popoverComponent: TbPopoverComponent; + + differ: AceDiff; + + contentReady = false; + + preferredDiffHeight = '332px'; + + isFullscreen = false; + + hasNext = false; + hasPrevious = false; + + diffCount = 0; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private elementRef: ElementRef, + private viewContainerRef: ViewContainerRef, + private popoverService: TbPopoverService) { + super(store); + } + + ngOnInit(): void { + this.entitiesVersionControlService + .compareEntityDataToVersion(this.branch, this.entityId, this.versionId).subscribe((diffData) => { + const leftContent = entityExportDataToJsonString(diffData.currentVersion); + const rightContent = entityExportDataToJsonString(diffData.otherVersion); + const leftLines = leftContent.split('\n').length; + const rightLines = leftContent.split('\n').length; + const totalLines = Math.max(leftLines, rightLines); + let preferredLines = Math.max(10, totalLines); + preferredLines = Math.min(40, preferredLines); + this.preferredDiffHeight = (132 + preferredLines * 16) + 'px'; + getAceDiff().subscribe((aceDiff) => { + this.contentReady = true; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + setTimeout(() => { + this.differ = new aceDiff.default( + { + element: this.diffViewerElmRef.nativeElement, + mode: 'ace/mode/json', + left: { + copyLinkEnabled: false, + editable: false, + content: leftContent + }, + right: { + copyLinkEnabled: false, + editable: false, + content: rightContent + } + } as AceDiff.AceDiffConstructorOpts + ); + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + leftEditor.setShowFoldWidgets(false); + rightEditor.setShowFoldWidgets(false); + $('.acediff__left .ace_scrollbar-v', this.elementRef.nativeElement).on('scroll', () => { + rightEditor.getSession().setScrollTop(leftEditor.getSession().getScrollTop()); + }); + $('.acediff__right .ace_scrollbar-v', this.elementRef.nativeElement).on('scroll', () => { + leftEditor.getSession().setScrollTop(rightEditor.getSession().getScrollTop()); + }); + $('.acediff__left .ace_scrollbar-h', this.elementRef.nativeElement).on('scroll', () => { + rightEditor.getSession().setScrollLeft(leftEditor.getSession().getScrollLeft()); + }); + $('.acediff__right .ace_scrollbar-h', this.elementRef.nativeElement).on('scroll', () => { + leftEditor.getSession().setScrollLeft(rightEditor.getSession().getScrollLeft()); + }); + leftEditor.getSession().getSelection().on('changeCursor', () => this.updateHasNextAndPrevious()); + rightEditor.getSession().getSelection().on('changeCursor', () => this.updateHasNextAndPrevious()); + setTimeout(() => { + this.diffCount = this.differ.getNumDiffs(); + this.updateHasNextAndPrevious(); + }); + }); + }); + }); + } + + versionIdContent(): string { + let versionId = this.versionId; + if (versionId.length > 7) { + versionId = versionId.slice(0, 7); + } + return versionId + ' (' + this.versionName + ')'; + } + + prevDifference($event: Event) { + if ($event) { + $event.stopPropagation(); + } + this.moveToDiff(false); + } + + nextDifference($event: Event) { + if ($event) { + $event.stopPropagation(); + } + this.moveToDiff(true); + } + + private moveToDiff(next: boolean) { + const currentRow = this.getCurrentRow(); + const diff = next ? this.findNextLine(currentRow) : this.findPrevLine(currentRow); + if (diff) { + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + leftEditor.scrollToLine(diff.leftStartLine + 1, true, true, () => {}); + leftEditor.gotoLine(diff.leftStartLine + 1, 0, true); + rightEditor.scrollToLine(diff.rightStartLine + 1, true, true, () => {}); + rightEditor.gotoLine(diff.rightStartLine + 1, 0, true); + } + } + + onFullscreenChanged(fullscreen: boolean) { + if (fullscreen) { + this.resizeEditors(); + } else { + setTimeout(() => { + this.resizeEditors(); + }); + } + } + + private getDiffs(): DiffInfo[] { + if (this.differ) { + // @ts-ignore + return this.differ.diffs as DiffInfo[] || []; + } else { + return []; + } + } + + private getCurrentRow(): {row: number, left: boolean} { + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + let currentRow = 0; + let left = true; + const leftRow = leftEditor.getSession().getSelection().getCursor().row; + const rightRow = rightEditor.getSession().getSelection().getCursor().row; + if (leftRow >= leftEditor.getFirstVisibleRow() && leftRow <= leftEditor.getLastVisibleRow()) { + currentRow = leftRow; + } else if (rightRow >= rightEditor.getFirstVisibleRow() && rightRow <= rightEditor.getLastVisibleRow()) { + currentRow = rightRow; + left = false; + } else { + currentRow = leftRow; + } + return {row: currentRow, left}; + } + + private nextDiff(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + const diffs = this.getDiffs(); + return diffs.find((diff) => (currentLine.left ? diff.leftStartLine : diff.rightStartLine) > currentLine.row); + } + + private prevDiff(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + const diffs = this.getDiffs(); + return [...diffs].reverse().find((diff) => (currentLine.left ? diff.leftEndLine : diff.rightEndLine) < currentLine.row); + } + + private findNextLine(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + let res = this.nextDiff(currentLine); + const diffs = this.getDiffs(); + if (!res && diffs.length) { + res = diffs[diffs.length - 1]; + } + return res; + } + + private findPrevLine(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + let res = this.prevDiff(currentLine); + const diffs = this.getDiffs(); + if (!res && diffs.length) { + res = diffs[0]; + } + return res; + } + + private updateHasNextAndPrevious() { + const currentRow = this.getCurrentRow(); + this.hasNext = !!this.nextDiff(currentRow); + this.hasPrevious = !!this.prevDiff(currentRow); + this.cd.markForCheck(); + } + + private resizeEditors() { + if (this.differ) { + this.differ.diff(); + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + leftEditor.resize(); + leftEditor.renderer.updateFull(); + rightEditor.resize(); + rightEditor.renderer.updateFull(); + } + } + + ngOnDestroy(): void { + if (this.differ) { + this.differ.destroy(); + this.differ = null; + } + } + + close(): void { + if (this.popoverComponent) { + this.popoverComponent.hide(); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: this.versionName, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + onClose: (result: VersionLoadResult | null) => { + restoreVersionPopover.hide(); + if (result && !result.error && result.result.length) { + this.close(); + this.versionRestored.emit(); + } + } + }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html new file mode 100644 index 0000000000..fa2d1f28ae --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -0,0 +1,69 @@ + +
+
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + + +
+
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+
+
+ +
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts new file mode 100644 index 0000000000..c31030f74b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -0,0 +1,121 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + EntityDataInfo, + SingleEntityVersionLoadRequest, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { delay } from 'rxjs/operators'; +import { SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-entity-version-restore', + templateUrl: './entity-version-restore.component.html', + styleUrls: ['./version-control.scss'] +}) +export class EntityVersionRestoreComponent extends PageComponent implements OnInit { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: (result: VersionLoadResult | null) => void; + + @Input() + popoverComponent: TbPopoverComponent; + + entityDataInfo: EntityDataInfo = null; + + restoreFormGroup: FormGroup; + + errorMessage: SafeHtml; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.restoreFormGroup = this.fb.group({ + loadAttributes: [true, []], + loadRelations: [true, []], + loadCredentials: [true, []] + }); + this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { + this.entityDataInfo = data; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + restore(): void { + const request: SingleEntityVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + config: { + loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, + loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false + }, + type: VersionLoadRequestType.SINGLE_ENTITY + }; + this.entitiesVersionControlService.loadEntitiesVersion(request).subscribe((result) => { + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else { + if (this.onClose) { + this.onClose(result); + } + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html new file mode 100644 index 0000000000..b6accc40dc --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -0,0 +1,172 @@ + +
+
+ +
+
+ {{(singleEntityMode ? 'version-control.entity-versions' : 'version-control.versions') | translate}} + + +
+ + + + + +
+ + +
+ + +   + + + +
+ +
+ + + {{ 'version-control.created-time' | translate }} + + {{ entityVersion.timestamp | date:'yyyy-MM-dd HH:mm:ss' }} + + + + {{ 'version-control.version-id' | translate }} + + + + + + + + {{ 'version-control.version-name' | translate }} + + {{ entityVersion.name }} + + + + {{ 'version-control.author' | translate }} + + {{ entityVersion.author }} + + + + + + +
+ + + +
+ + + + +
+ {{ + singleEntityMode + ? 'version-control.no-entity-versions-text' + : 'version-control.no-versions-text' + }} + {{ 'common.loading' | translate }} +
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss new file mode 100644 index 0000000000..3256c3c956 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -0,0 +1,107 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../scss/constants'; + +:host { + width: 100%; + height: 100%; + display: block; + .tb-entity-table { + + &.tb-popover-mode { + position: relative; + width: 800px; + height: 600px; + } + + .tb-entity-table-content { + width: 100%; + height: 100%; + background: #fff; + + .mat-toolbar-tools{ + min-height: auto; + } + + .title-container{ + overflow: hidden; + } + + .tb-entity-table-title { + padding-right: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .table-container { + overflow: auto; + } + + .tb-entity-table-info{ + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .button-widget-action{ + margin-left: auto; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + @media #{$mat-xs} { + .mat-toolbar { + height: auto; + min-height: 100px; + + .tb-entity-table-title{ + padding-bottom: 5px; + width: 100%; + } + } + } +} + +:host ::ng-deep { + .mat-sort-header-sorted .mat-sort-header-arrow { + opacity: 1 !important; + } + tb-branch-autocomplete { + mat-form-field { + font-size: 16px; + width: 250px; + + .mat-form-field-wrapper { + padding-bottom: 0; + } + + .mat-form-field-underline { + bottom: 0; + } + + @media #{$mat-xs} { + width: 100%; + + .mat-form-field-infix { + width: auto !important; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts new file mode 100644 index 0000000000..d776939aaf --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -0,0 +1,445 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, EventEmitter, + Input, + OnDestroy, + OnInit, Output, Renderer2, + ViewChild, ViewContainerRef +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityId, entityIdEquals } from '@shared/models/id/entity-id'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; +import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; +import { EntityVersion, VersionCreationResult, VersionLoadResult } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatSort } from '@angular/material/sort'; +import { ResizeObserver } from '@juggle/resize-observer'; +import { hidePageSizePixelValue } from '@shared/models/constants'; +import { Direction, SortOrder } from '@shared/models/page/sort-order'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; +import { isNotEmptyStr } from '@core/utils'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { MatButton } from '@angular/material/button'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; +import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; +import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-entity-versions-table', + templateUrl: './entity-versions-table.component.html', + styleUrls: ['./entity-versions-table.component.scss'] +}) +export class EntityVersionsTableComponent extends PageComponent implements OnInit, AfterViewInit, OnDestroy { + + @ViewChild('branchAutocompleteComponent') branchAutocompleteComponent: BranchAutocompleteComponent; + + @Input() + singleEntityMode = false; + + @Input() + popoverComponent: TbPopoverComponent; + + @Input() + onBeforeCreateVersion: () => Observable; + + displayedColumns = ['timestamp', 'id', 'name', 'author', 'actions']; + pageLink: PageLink; + textSearchMode = false; + dataSource: EntityVersionsDatasource; + hidePageSize = false; + + branch: string = null; + + activeValue = false; + dirtyValue = false; + externalEntityIdValue: EntityId; + + viewsInited = false; + + private componentResize$: ResizeObserver; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + if (this.viewsInited) { + this.initFromDefaultBranch(); + } + } + } + } + + @Input() + set externalEntityId(externalEntityId: EntityId) { + if (!entityIdEquals(this.externalEntityIdValue, externalEntityId)) { + this.externalEntityIdValue = externalEntityId; + this.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.dirtyValue = true; + } + } + } + + @Input() + entityId: EntityId; + + @Input() + entityName: string; + + @Output() + versionRestored = new EventEmitter(); + + @ViewChild('searchInput') searchInputField: ElementRef; + + @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private cd: ChangeDetectorRef, + private viewContainerRef: ViewContainerRef, + private elementRef: ElementRef) { + super(store); + this.dirtyValue = !this.activeValue; + const sortOrder: SortOrder = { property: 'timestamp', direction: Direction.DESC }; + this.pageLink = new PageLink(10, 0, null, sortOrder); + this.dataSource = new EntityVersionsDatasource(this.entitiesVersionControlService); + } + + ngOnInit() { + this.componentResize$ = new ResizeObserver(() => { + const showHidePageSize = this.elementRef.nativeElement.offsetWidth < hidePageSizePixelValue; + if (showHidePageSize !== this.hidePageSize) { + this.hidePageSize = showHidePageSize; + this.cd.markForCheck(); + } + }); + this.componentResize$.observe(this.elementRef.nativeElement); + } + + ngOnDestroy() { + if (this.componentResize$) { + this.componentResize$.disconnect(); + } + } + + branchChanged(newBranch: string) { + if (isNotEmptyStr(newBranch) && this.branch !== newBranch) { + this.branch = newBranch; + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } + } + + ngAfterViewInit() { + fromEvent(this.searchInputField.nativeElement, 'keyup') + .pipe( + debounceTime(400), + distinctUntilChanged(), + tap(() => { + this.paginator.pageIndex = 0; + this.updateData(); + }) + ) + .subscribe(); + + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); + merge(this.sort.sortChange, this.paginator.page) + .pipe( + tap(() => this.updateData()) + ) + .subscribe(); + this.viewsInited = true; + if (!this.singleEntityMode || (this.activeValue && this.externalEntityIdValue)) { + this.initFromDefaultBranch(); + } + } + + toggleCreateVersion($event: Event, createVersionButton: MatButton) { + if ($event) { + $event.stopPropagation(); + } + const trigger = createVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const createVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionCreateComponent, 'leftTop', true, null, + { + branch: this.branch, + entityId: this.entityId, + entityName: this.entityName, + onBeforeCreateVersion: this.onBeforeCreateVersion, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + createVersionPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + createVersionPopover.tbComponentRef.instance.popoverComponent = createVersionPopover; + } + } + + toggleComplexCreateVersion($event: Event, complexCreateVersionButton: MatButton) { + if ($event) { + $event.stopPropagation(); + } + const trigger = complexCreateVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const complexCreateVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionCreateComponent, 'leftTop', true, null, + { + branch: this.branch, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + complexCreateVersionPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + complexCreateVersionPopover.tbComponentRef.instance.popoverComponent = complexCreateVersionPopover; + } + } + + toggleShowVersionDiff($event: Event, diffVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = diffVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const diffVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionDiffComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + entityId: this.entityId, + externalEntityId: this.externalEntityIdValue + }, {}, {}, {}, false); + diffVersionPopover.tbComponentRef.instance.popoverComponent = diffVersionPopover; + diffVersionPopover.tbComponentRef.instance.versionRestored.subscribe(() => { + this.versionRestored.emit(); + }); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + externalEntityId: this.externalEntityIdValue, + onClose: (result: VersionLoadResult | null) => { + restoreVersionPopover.hide(); + if (result && !result.error && result.result.length) { + this.versionRestored.emit(); + } + } + }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; + } + } + + toggleRestoreEntitiesVersion($event: Event, restoreEntitiesVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreEntitiesVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreEntitiesVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionLoadComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + onClose: (result: VersionLoadResult | null) => { + restoreEntitiesVersionPopover.hide(); + } + }, {}, {}, {}, false); + restoreEntitiesVersionPopover.tbComponentRef.instance.popoverComponent = restoreEntitiesVersionPopover; + } + } + + versionIdContent(entityVersion: EntityVersion): string { + let versionId = entityVersion.id; + if (versionId.length > 7) { + versionId = versionId.slice(0, 7); + } + return versionId; + } + + enterFilterMode() { + this.textSearchMode = true; + this.pageLink.textSearch = ''; + setTimeout(() => { + this.searchInputField.nativeElement.focus(); + this.searchInputField.nativeElement.setSelectionRange(0, 0); + }, 10); + } + + exitFilterMode() { + this.textSearchMode = false; + this.pageLink.textSearch = null; + this.paginator.pageIndex = 0; + this.updateData(); + } + + private initFromDefaultBranch() { + if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } else { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(true); + } + } + + updateData() { + this.pageLink.page = this.paginator.pageIndex; + this.pageLink.pageSize = this.paginator.pageSize; + this.pageLink.sortOrder.property = this.sort.active; + this.pageLink.sortOrder.direction = Direction[this.sort.direction.toUpperCase()]; + this.dataSource.loadEntityVersions(this.singleEntityMode, this.branch, this.externalEntityIdValue, this.pageLink); + } + + private resetSortAndFilter(update: boolean) { + this.textSearchMode = false; + this.pageLink.textSearch = null; + if (this.viewsInited) { + this.paginator.pageIndex = 0; + const sortable = this.sort.sortables.get('timestamp'); + this.sort.active = sortable.id; + this.sort.direction = 'desc'; + if (update) { + this.initFromDefaultBranch(); + } + } + } +} + +class EntityVersionsDatasource implements DataSource { + + private entityVersionsSubject = new BehaviorSubject([]); + private pageDataSubject = new BehaviorSubject>(emptyPageData()); + + public pageData$ = this.pageDataSubject.asObservable(); + + public dataLoading = true; + + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} + + connect(collectionViewer: CollectionViewer): Observable> { + return this.entityVersionsSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entityVersionsSubject.complete(); + this.pageDataSubject.complete(); + } + + loadEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + this.dataLoading = true; + const result = new ReplaySubject>(); + this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entityVersionsSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + this.dataLoading = false; + } + ); + return result; + } + + fetchEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + if (!branch) { + return of(emptyPageData()); + } else { + if (singleEntityMode) { + if (externalEntityId) { + return this.entitiesVersionControlService.listEntityVersions(pageLink, branch, externalEntityId, {ignoreErrors: true}); + } else { + return of(emptyPageData()); + } + } else { + return this.entitiesVersionControlService.listVersions(pageLink, branch, {ignoreErrors: true}); + } + } + } + + isEmpty(): Observable { + return this.entityVersionsSubject.pipe( + map((entityVersions) => !entityVersions.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html new file mode 100644 index 0000000000..14ae91e2f2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html @@ -0,0 +1,41 @@ + +
+
+
+
+ + + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts new file mode 100644 index 0000000000..f569d96fac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts @@ -0,0 +1,66 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-remove-other-entities-confirm', + templateUrl: './remove-other-entities-confirm.component.html', + styleUrls: [] +}) +export class RemoveOtherEntitiesConfirmComponent extends PageComponent implements OnInit { + + @Input() + onClose: (result: boolean | null) => void; + + confirmFormGroup: FormGroup; + + removeOtherEntitiesConfirmText: SafeHtml; + + removeOtherEntitiesVerificationText = 'remove other entities'; + + constructor(protected store: Store, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + this.removeOtherEntitiesConfirmText = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.remove-other-entities-confirm-text')); + } + + ngOnInit(): void { + this.confirmFormGroup = this.fb.group({ + verification: [null, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + confirm(): void { + if (this.onClose) { + this.onClose(true); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html new file mode 100644 index 0000000000..3e8bd271f3 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -0,0 +1,110 @@ + +
+ + +
+ admin.repository-settings + +
+
+ + + +
+ +
+
+ + admin.repository-url + + + admin.repository-url-required + + + + admin.default-branch + + +
+ admin.authentication-settings + + admin.auth-method + + + {{repositoryAuthMethodTranslations.get(method) | translate}} + + + +
+ + common.username + + + + {{ 'admin.change-password-access-token' | translate }} + + + admin.password-access-token + + + +
+
+ + + + {{ 'admin.change-passphrase' | translate }} + + + admin.passphrase + + + +
+
+
+ + + + +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss new file mode 100644 index 0000000000..77ae0ffd8f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.repository-settings { + margin: 8px; + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts new file mode 100644 index 0000000000..1120185db9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -0,0 +1,221 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + RepositorySettings, + RepositoryAuthMethod, + repositoryAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; +import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { catchError, mergeMap, take } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-repository-settings', + templateUrl: './repository-settings.component.html', + styleUrls: ['./repository-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class RepositorySettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; + + @Input() + popoverComponent: TbPopoverComponent; + + repositorySettingsForm: FormGroup; + settings: RepositorySettings = null; + + repositoryAuthMethod = RepositoryAuthMethod; + repositoryAuthMethods = Object.values(RepositoryAuthMethod); + repositoryAuthMethodTranslations = repositoryAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + private cd: ChangeDetectorRef, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.repositorySettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: ['main', []], + authMethod: [RepositoryAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.repositorySettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.repositorySettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.store.pipe( + select(selectHasRepository), + take(1), + mergeMap((hasRepository) => { + if (hasRepository) { + return this.adminService.getRepositorySettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + if (this.settings != null) { + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + } + }); + } + + checkAccess(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.checkRepositoryAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-repository-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.saveRepositorySettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: true })); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-repository-settings-title', ), + this.translate.instant('admin.delete-repository-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteRepositorySettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.repositorySettingsForm.reset({ defaultBranch: 'main', authMethod: RepositoryAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: false })); + } + ); + } + }); + } + + changePasswordChanged() { + if (this.changePassword) { + this.repositorySettingsForm.get('password').patchValue(''); + this.repositorySettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').patchValue(''); + this.repositorySettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: RepositoryAuthMethod = this.repositorySettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.repositorySettingsForm.get('privateKeyFileName').value; + if (authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.repositorySettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.repositorySettingsForm.get('password').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('password').disable({emitEvent}); + } + this.repositorySettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.repositorySettingsForm.get('privateKey').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.repositorySettingsForm.get('username').disable({emitEvent}); + this.repositorySettingsForm.get('password').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.repositorySettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.repositorySettingsForm.get('privateKey').clearValidators(); + } else { + this.repositorySettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.repositorySettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html new file mode 100644 index 0000000000..eaf80fd664 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -0,0 +1,31 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss new file mode 100644 index 0000000000..da8df4b469 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts new file mode 100644 index 0000000000..59816b7b5d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; +import { FormGroup } from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-version-control', + templateUrl: './version-control.component.html', + styleUrls: ['./version-control.component.scss'] +}) +export class VersionControlComponent implements OnInit, HasConfirmForm { + + @ViewChild('repositorySettingsComponent', {static: false}) repositorySettingsComponent: RepositorySettingsComponent; + + @Input() + detailsMode = false; + + @Input() + popoverComponent: TbPopoverComponent; + + @Input() + active = true; + + @Input() + singleEntityMode = false; + + @Input() + externalEntityId: EntityId; + + @Input() + entityId: EntityId; + + @Input() + entityName: string; + + @Input() + onBeforeCreateVersion: () => Observable; + + @Output() + versionRestored = new EventEmitter(); + + hasRepository$ = this.store.pipe(select(selectHasRepository)); + + constructor(private store: Store) { + + } + + ngOnInit() { + + } + + confirmForm(): FormGroup { + return this.repositorySettingsComponent?.repositorySettingsForm; + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.scss new file mode 100644 index 0000000000..01aaae5455 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.scss @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .vc-result-message { + padding: 0 8px; + text-align: center; + &:first-child { + padding-top: 48px; + } + &:nth-last-child(2) { + padding-bottom: 8px; + } + &.error { + text-align: start; + font-weight: 400; + } + } +} diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 6dc8746b2e..ff08af8039 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -53,4 +53,5 @@ export class HomeDialogsService { } }).afterClosed(); } + } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index e74bb0ae4d..84020bc491 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,6 +33,8 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; +import { AutoCommitAdminSettingsComponent } from '@home/pages/admin/auto-commit-admin-settings.component'; import { TwoFactorAuthSettingsComponent } from '@home/pages/admin/two-factor-auth-settings.component'; @Injectable() @@ -237,6 +239,32 @@ const routes: Routes = [ isMdiIcon: true } } + }, + { + path: 'repository', + component: RepositoryAdminSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.repository-settings', + breadcrumb: { + label: 'admin.repository-settings', + icon: 'manage_history' + } + } + }, + { + path: 'auto-commit', + component: AutoCommitAdminSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.auto-commit-settings', + breadcrumb: { + label: 'admin.auto-commit-settings', + icon: 'settings_backup_restore' + } + } } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index a6e9393611..54ffbc61b2 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,6 +29,8 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; +import { AutoCommitAdminSettingsComponent } from '@home/pages/admin/auto-commit-admin-settings.component'; import { TwoFactorAuthSettingsComponent } from '@home/pages/admin/two-factor-auth-settings.component'; @NgModule({ @@ -43,6 +45,8 @@ import { TwoFactorAuthSettingsComponent } from '@home/pages/admin/two-factor-aut HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, + RepositoryAdminSettingsComponent, + AutoCommitAdminSettingsComponent, TwoFactorAuthSettingsComponent ], imports: [ diff --git a/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html new file mode 100644 index 0000000000..63060f6a0a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html @@ -0,0 +1,23 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts new file mode 100644 index 0000000000..6c5c110884 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts @@ -0,0 +1,51 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { AutoCommitSettingsComponent } from '@home/components/vc/auto-commit-settings.component'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; + +@Component({ + selector: 'tb-auto-commit-admin-settings', + templateUrl: './auto-commit-admin-settings.component.html', + styleUrls: [] +}) +export class AutoCommitAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('repositorySettingsComponent', {static: false}) repositorySettingsComponent: RepositorySettingsComponent; + @ViewChild('autoCommitSettingsComponent', {static: false}) autoCommitSettingsComponent: AutoCommitSettingsComponent; + + hasRepository$ = this.store.pipe(select(selectHasRepository)); + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.repositorySettingsComponent ? + this.repositorySettingsComponent?.repositorySettingsForm : + this.autoCommitSettingsComponent?.autoCommitSettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html new file mode 100644 index 0000000000..7b408fc93a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html @@ -0,0 +1,18 @@ + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts new file mode 100644 index 0000000000..1ee0a7288a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; + +@Component({ + selector: 'tb-repository-admin-settings', + templateUrl: './repository-admin-settings.component.html', + styleUrls: [] +}) +export class RepositoryAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('repositorySettingsComponent') repositorySettingsComponent: RepositorySettingsComponent; + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.repositorySettingsComponent?.repositorySettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index e5accc15a9..452dce8bcd 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html index 9c97f1d96d..942ea2a73b 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts index a3716823e1..bbdde763e2 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts @@ -34,6 +34,7 @@ import { CustomerTabsComponent } from '@home/pages/customer/customer-tabs.compon import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; +import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; @Injectable() export class CustomersTableConfigResolver implements Resolve> { @@ -41,6 +42,7 @@ export class CustomersTableConfigResolver implements Resolve = new EntityTableConfig(); constructor(private customerService: CustomerService, + private homeDialogs: HomeDialogsService, private translate: TranslateService, private datePipe: DatePipe, private router: Router, diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html index 0e6fa55e8c..a9fc2f644c 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html @@ -19,3 +19,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts index f51043fb3b..fddc9e94ce 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts @@ -69,6 +69,7 @@ import { AddEntitiesToEdgeDialogComponent, AddEntitiesToEdgeDialogData } from '@home/dialogs/add-entities-to-edge-dialog.component'; +import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; @Injectable() export class DashboardsTableConfigResolver implements Resolve> { @@ -80,6 +81,7 @@ export class DashboardsTableConfigResolver implements Resolve + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts index 2434f50151..597e2a6390 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts @@ -42,6 +42,7 @@ import { AddDeviceProfileDialogData } from '@home/components/profile/add-device-profile-dialog.component'; import { ImportExportService } from '@home/components/import-export/import-export.service'; +import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; @Injectable() export class DeviceProfilesTableConfigResolver implements Resolve> { @@ -50,6 +51,7 @@ export class DeviceProfilesTableConfigResolver implements Resolve + + + diff --git a/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html b/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html index 7ba3c80805..79307ddcb2 100644 --- a/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts index cf3aec6191..1b45d32a63 100644 --- a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts @@ -37,6 +37,7 @@ import { DeviceProfileModule } from './device-profile/device-profile.module'; import { ApiUsageModule } from '@home/pages/api-usage/api-usage.module'; import { EdgeModule } from '@home/pages/edge/edge.module'; import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; +import { VcModule } from '@home/pages/vc/vc.module'; @NgModule({ exports: [ @@ -58,7 +59,8 @@ import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; AuditLogModule, ApiUsageModule, OtaUpdateModule, - UserModule + UserModule, + VcModule ], providers: [ { diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html index 49eb3953f5..84d768c61d 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html @@ -164,6 +164,17 @@ + + + + check + + + {{ 'version-control.default' | translate }} + + + + {{ 'version-control.branch-required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.scss b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.scss new file mode 100644 index 0000000000..06b464db27 --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.scss @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.mat-option.branch-option { + .mat-icon, .check-placeholder { + margin-right: 8px; + } + .check-placeholder { + width: 18px; + display: inline-block; + } + .mat-option-text { + width: 100%; + .default-branch { + float: right; + } + } +} diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts new file mode 100644 index 0000000000..c135b0a8f7 --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -0,0 +1,296 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + ElementRef, + forwardRef, + Input, + NgZone, + OnInit, + ViewChild, + ViewEncapsulation +} from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable } from 'rxjs'; +import { debounceTime, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { BranchInfo } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { isNotEmptyStr } from '@core/utils'; +import { MatAutocomplete, MatAutocompleteTrigger } from '@angular/material/autocomplete'; + +@Component({ + selector: 'tb-branch-autocomplete', + templateUrl: './branch-autocomplete.component.html', + styleUrls: ['./branch-autocomplete.component.scss'], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => BranchAutocompleteComponent), + multi: true + }], + encapsulation: ViewEncapsulation.None +}) +export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit, AfterViewInit { + + branchFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + private disabledValue: boolean; + + get disabled(): boolean { + return this.disabledValue; + } + + @Input() + set disabled(value: boolean) { + this.disabledValue = coerceBooleanProperty(value); + if (this.disabledValue) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } + + @Input() + selectDefaultBranch = true; + + @Input() + selectionMode = false; + + @Input() + emptyPlaceholder: string; + + @ViewChild('branchAutocomplete') matAutocomplete: MatAutocomplete; + @ViewChild('branchInput', { read: MatAutocompleteTrigger, static: true }) autoCompleteTrigger: MatAutocompleteTrigger; + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; + + filteredBranches: Observable>; + + defaultBranch: BranchInfo = null; + + searchText = ''; + + loading = false; + + private dirty = false; + + private clearButtonClicked = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder, + private zone: NgZone) { + this.branchFormGroup = this.fb.group({ + branch: [null, []] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.filteredBranches = this.branchFormGroup.get('branch').valueChanges + .pipe( + tap((value: BranchInfo | string) => { + let modelValue: BranchInfo | null; + if (typeof value === 'string' || !value) { + if (!this.selectionMode && typeof value === 'string' && isNotEmptyStr(value)) { + modelValue = {name: value, default: false}; + } else { + modelValue = null; + } + } else { + modelValue = value; + } + if (!this.selectionMode || modelValue) { + this.updateView(modelValue); + } + }), + map(value => { + if (value) { + if (typeof value === 'string') { + return value; + } else { + return value.name; + } + } else { + return ''; + } + }), + debounceTime(150), + distinctUntilChanged(), + switchMap(name => this.fetchBranches(name)), + share() + ); + } + + ngAfterViewInit(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + } + + isDefaultBranchSelected(): boolean { + return this.defaultBranch && this.defaultBranch.name === this.modelValue; + } + + selectDefaultBranchIfNeeded(force = false): void { + if ((this.selectDefaultBranch && !this.modelValue) || force) { + setTimeout(() => { + if (this.defaultBranch) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); + } else { + this.loading = true; + this.getBranches().subscribe( + () => { + if (this.defaultBranch || force) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); + this.loading = false; + } else { + this.loading = false; + } + } + ); + } + }); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + if (value != null) { + this.branchFormGroup.get('branch').patchValue({name: value}, {emitEvent: false}); + } else { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: false}); + this.selectDefaultBranchIfNeeded(); + } + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.branchFormGroup.get('branch').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + onBlur() { + if (this.clearButtonClicked) { + this.clearButtonClicked = false; + } else if (!this.matAutocomplete.isOpen) { + this.selectAvailableValue(); + } + } + + onPanelClosed() { + this.selectAvailableValue(); + } + + selectAvailableValue() { + if (this.selectionMode) { + const branch = this.branchFormGroup.get('branch').value; + this.getBranches().pipe( + map(branches => { + let foundBranch = branches.find(b => b.name === branch); + if (!foundBranch && isNotEmptyStr(this.modelValue)) { + foundBranch = branches.find(b => b.name === this.modelValue); + } + return foundBranch; + }) + ).subscribe((val) => { + if (!val && this.defaultBranch) { + val = this.defaultBranch; + } + this.zone.run(() => { + this.branchFormGroup.get('branch').patchValue(val, {emitEvent: true}); + }, 0); + }); + } + } + + updateView(value: BranchInfo | null) { + if (this.modelValue !== value?.name) { + this.modelValue = value?.name; + this.propagateChange(this.modelValue); + } + } + + displayBranchFn(branch?: BranchInfo): string | undefined { + return branch ? branch.name : undefined; + } + + private fetchBranches(searchText?: string): Observable> { + this.searchText = searchText; + return this.getBranches().pipe( + map(branches => { + let res = branches.filter(branch => { + return searchText ? branch.name.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (!this.selectionMode && isNotEmptyStr(searchText) && !res.find(b => b.name === searchText)) { + res = [{name: searchText, default: false}, ...res]; + } + return res; + } + ) + ); + } + + private getBranches(): Observable> { + return this.entitiesVersionControlService.listBranches().pipe( + tap((data) => { + this.defaultBranch = data.find(branch => branch.default); + }) + ); + } + + clear() { + this.clearButtonClicked = true; + setTimeout(() => { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); + this.branchInput.nativeElement.blur(); + this.branchInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/ace/ace.models.ts b/ui-ngx/src/app/shared/models/ace/ace.models.ts index 835c8d1cfa..9777935922 100644 --- a/ui-ngx/src/app/shared/models/ace/ace.models.ts +++ b/ui-ngx/src/app/shared/models/ace/ace.models.ts @@ -21,6 +21,7 @@ import { map, mergeMap, tap } from 'rxjs/operators'; let aceDependenciesLoaded = false; let aceModule: any; +let aceDiffModule: any; function loadAceDependencies(): Observable { if (aceDependenciesLoaded) { @@ -74,6 +75,21 @@ export function getAce(): Observable { } } +export function getAceDiff(): Observable { + if (aceDiffModule) { + return of(aceDiffModule); + } else { + return getAce().pipe( + mergeMap((ace) => { + return from(import('ace-diff')); + }), + tap((module) => { + aceDiffModule = module; + }) + ); + } +} + export class Range implements Ace.Range { public start: Ace.Point; diff --git a/ui-ngx/src/app/shared/models/asset.models.ts b/ui-ngx/src/app/shared/models/asset.models.ts index d38aee4d96..115eea4610 100644 --- a/ui-ngx/src/app/shared/models/asset.models.ts +++ b/ui-ngx/src/app/shared/models/asset.models.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { AssetId } from './id/asset-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; -export interface Asset extends BaseData { +export interface Asset extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/base-data.ts b/ui-ngx/src/app/shared/models/base-data.ts index 3af9bb9059..7b3ea9a642 100644 --- a/ui-ngx/src/app/shared/models/base-data.ts +++ b/ui-ngx/src/app/shared/models/base-data.ts @@ -27,6 +27,12 @@ export interface BaseData { label?: string; } +export interface ExportableEntity { + createdTime?: number; + id?: T; + externalId?: T; +} + export function hasIdEquals(id1: HasId, id2: HasId): boolean { if (isDefinedAndNotNull(id1) && isDefinedAndNotNull(id2)) { return id1.id === id2.id; diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 37027edc68..b746ee71b0 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -135,7 +135,9 @@ export const HelpLinks = { widgetsConfigStatic: helpBaseUrl + '/docs/user-guide/ui/dashboards#static', ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', - queue: helpBaseUrl + '/docs/user-guide/queue' + queue: helpBaseUrl + '/docs/user-guide/queue', + repositorySettings: helpBaseUrl + '/docs/user-guide/ui/repository-settings', + autoCommitSettings: helpBaseUrl + '/docs/user-guide/ui/auto-commit-settings', } }; diff --git a/ui-ngx/src/app/shared/models/customer.model.ts b/ui-ngx/src/app/shared/models/customer.model.ts index 435edf263f..67c1429189 100644 --- a/ui-ngx/src/app/shared/models/customer.model.ts +++ b/ui-ngx/src/app/shared/models/customer.model.ts @@ -17,8 +17,9 @@ import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; import { TenantId } from './id/tenant-id'; +import { ExportableEntity } from '@shared/models/base-data'; -export interface Customer extends ContactBased { +export interface Customer extends ContactBased, ExportableEntity { tenantId: TenantId; title: string; additionalInfo?: any; diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index 4332e484f6..12e1c83ada 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DashboardId } from '@shared/models/id/dashboard-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { ShortCustomerInfo } from '@shared/models/customer.model'; @@ -23,7 +23,7 @@ import { Timewindow } from '@shared/models/time/time.models'; import { EntityAliases } from './alias.models'; import { Filters } from '@shared/models/query/query.models'; -export interface DashboardInfo extends BaseData { +export interface DashboardInfo extends BaseData, ExportableEntity { tenantId?: TenantId; title?: string; image?: string; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 1d6ffa13af..88a2ccf16e 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DeviceId } from './id/device-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; @@ -564,7 +564,7 @@ export interface DeviceProfileData { provisionConfiguration?: DeviceProvisionConfiguration; } -export interface DeviceProfile extends BaseData { +export interface DeviceProfile extends BaseData, ExportableEntity { tenantId?: TenantId; name: string; description?: string; @@ -689,7 +689,7 @@ export interface DeviceData { transportConfiguration: DeviceTransportConfiguration; } -export interface Device extends BaseData { +export interface Device extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index c92c2c43db..65753f3962 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -251,6 +251,9 @@ export const entityTypeTranslations = new Map { +export interface EntityView extends BaseData, ExportableEntity { tenantId: TenantId; customerId: CustomerId; entityId: EntityId; diff --git a/ui-ngx/src/app/shared/models/rule-chain.models.ts b/ui-ngx/src/app/shared/models/rule-chain.models.ts index 85d6047191..01f9d268c3 100644 --- a/ui-ngx/src/app/shared/models/rule-chain.models.ts +++ b/ui-ngx/src/app/shared/models/rule-chain.models.ts @@ -14,14 +14,14 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainId } from '@shared/models/id/rule-chain-id'; import { RuleNodeId } from '@shared/models/id/rule-node-id'; import { RuleNode, RuleNodeComponentDescriptor, RuleNodeType } from '@shared/models/rule-node.models'; import { ComponentType } from '@shared/models/component-descriptor.models'; -export interface RuleChain extends BaseData { +export interface RuleChain extends BaseData, ExportableEntity { tenantId: TenantId; name: string; firstRuleNodeId: RuleNodeId; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index e00c8e28a0..c1a7cae87a 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -16,6 +16,7 @@ import { ValidatorFn } from '@angular/forms'; import { isNotEmptyStr, isNumber } from '@core/utils'; +import { VersionCreateConfig } from '@shared/models/vc.models'; export const smtpPortPattern: RegExp = /^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/; @@ -396,3 +397,30 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid } return smsProviderConfiguration; } + +export enum RepositoryAuthMethod { + USERNAME_PASSWORD = 'USERNAME_PASSWORD', + PRIVATE_KEY = 'PRIVATE_KEY' +} + +export const repositoryAuthMethodTranslationMap = new Map([ + [RepositoryAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [RepositoryAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +]); + +export interface RepositorySettings { + repositoryUri: string; + defaultBranch: string; + authMethod: RepositoryAuthMethod; + username: string; + password: string; + privateKeyFileName: string; + privateKey: string; + privateKeyPassword: string; +} + +export interface AutoVersionCreateConfig extends VersionCreateConfig { + branch: string; +} + +export type AutoCommitSettings = {[entityType: string]: AutoVersionCreateConfig}; diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 5f91707a26..70bbb1dc28 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -41,6 +41,9 @@ export interface DefaultTenantProfileConfiguration { transportDeviceTelemetryMsgRateLimit?: string; transportDeviceTelemetryDataPointsRateLimit?: string; + tenantEntityExportRateLimit?: string; + tenantEntityImportRateLimit?: string; + maxTransportMessages: number; maxTransportDataPoints: number; maxREExecutions: number; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts new file mode 100644 index 0000000000..5ef0a18b08 --- /dev/null +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -0,0 +1,239 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; +import { ExportableEntity } from '@shared/models/base-data'; +import { EntityRelation } from '@shared/models/relation.models'; +import { Device, DeviceCredentials } from '@shared/models/device.models'; +import { RuleChain, RuleChainMetaData } from '@shared/models/rule-chain.models'; + +export const exportableEntityTypes: Array = [ + EntityType.ASSET, + EntityType.DEVICE, + EntityType.ENTITY_VIEW, + EntityType.DASHBOARD, + EntityType.CUSTOMER, + EntityType.DEVICE_PROFILE, + EntityType.RULE_CHAIN, + EntityType.WIDGETS_BUNDLE +]; + +export interface VersionCreateConfig { + saveRelations: boolean; + saveAttributes: boolean; + saveCredentials: boolean; +} + +export enum VersionCreateRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + COMPLEX = 'COMPLEX' +} + +export interface VersionCreateRequest { + versionName: string; + branch: string; + type: VersionCreateRequestType; +} + +export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { + entityId: EntityId; + config: VersionCreateConfig; + type: VersionCreateRequestType.SINGLE_ENTITY; +} + +export enum SyncStrategy { + MERGE = 'MERGE', + OVERWRITE = 'OVERWRITE' +} + +export const syncStrategyTranslationMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite'] + ] +); + +export const syncStrategyHintMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge-hint'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite-hint'] + ] +); + +export interface EntityTypeVersionCreateConfig extends VersionCreateConfig { + syncStrategy: SyncStrategy; + entityIds: string[]; + allEntities: boolean; +} + +export interface ComplexVersionCreateRequest extends VersionCreateRequest { + syncStrategy: SyncStrategy; + entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig}; + type: VersionCreateRequestType.COMPLEX; +} + +export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: EntityTypeVersionCreateConfig} { + const res: {[entityType: string]: EntityTypeVersionCreateConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + syncStrategy: null, + saveAttributes: true, + saveRelations: true, + saveCredentials: true, + allEntities: true, + entityIds: [] + }; + } + return res; +} + +export interface VersionLoadConfig { + loadRelations: boolean; + loadAttributes: boolean; + loadCredentials: boolean; +} + +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + +export interface VersionLoadRequest { + branch: string; + versionId: string; + type: VersionLoadRequestType; +} + +export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { + externalEntityId: EntityId; + config: VersionLoadConfig; + type: VersionLoadRequestType.SINGLE_ENTITY; +} + +export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { + removeOtherEntities: boolean; + findExistingEntityByName: boolean; +} + +export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { + entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig}; + type: VersionLoadRequestType.ENTITY_TYPE; +} + +export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} { + const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + loadAttributes: true, + loadRelations: true, + loadCredentials: true, + removeOtherEntities: false, + findExistingEntityByName: true + }; + } + return res; +} + +export interface BranchInfo { + name: string; + default: boolean; +} + +export interface EntityVersion { + timestamp: number; + id: string; + name: string; + author: string; +} + +export interface VersionCreationResult { + version: EntityVersion; + added: number; + modified: number; + removed: number; +} + +export interface EntityTypeLoadResult { + entityType: EntityType; + created: number; + updated: number; + deleted: number; +} + +export enum EntityLoadErrorType { + DEVICE_CREDENTIALS_CONFLICT = 'DEVICE_CREDENTIALS_CONFLICT', + MISSING_REFERENCED_ENTITY = 'MISSING_REFERENCED_ENTITY' +} + +export const entityLoadErrorTranslationMap = new Map( + [ + [EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT, 'version-control.device-credentials-conflict'], + [EntityLoadErrorType.MISSING_REFERENCED_ENTITY, 'version-control.missing-referenced-entity'] + ] +); + +export interface EntityLoadError { + type: EntityLoadErrorType; + source: EntityId; + target: EntityId; +} + +export interface VersionLoadResult { + result: Array; + error: EntityLoadError; +} + +export interface AttributeExportData { + key: string; + lastUpdateTs: number; + booleanValue: boolean; + strValue: string; + longValue: number; + doubleValue: number; + jsonValue: string; +} + +export interface EntityExportData> { + entity: E; + entityType: EntityType; + relations: Array; + attributes: {[key: string]: Array}; +} + +export interface DeviceExportData extends EntityExportData { + credentials: DeviceCredentials; +} + +export interface RuleChainExportData extends EntityExportData { + metaData: RuleChainMetaData; +} + +export interface EntityDataDiff { + currentVersion: EntityExportData; + otherVersion: EntityExportData; + rawDiff: string; +} + +export function entityExportDataToJsonString(data: EntityExportData): string { + return JSON.stringify(data, null, 4); +} + +export interface EntityDataInfo { + hasRelations: boolean; + hasAttributes: boolean; + hasCredentials: boolean; +} diff --git a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts index 106438a591..235190ead2 100644 --- a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts +++ b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetsBundleId } from '@shared/models/id/widgets-bundle-id'; -export interface WidgetsBundle extends BaseData { +export interface WidgetsBundle extends BaseData, ExportableEntity { tenantId: TenantId; alias: string; title: string; diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 81f2ff0afb..aaa29f02c0 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -163,6 +163,7 @@ import { HtmlComponent } from '@shared/components/html.component'; import { SafePipe } from '@shared/pipe/safe.pipe'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { MultipleImageInputComponent } from '@shared/components/multiple-image-input.component'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; import { PhoneInputComponent } from '@shared/components/phone-input.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { @@ -286,6 +287,7 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) CopyButtonComponent, TogglePasswordComponent, ProtobufContentComponent, + BranchAutocompleteComponent, PhoneInputComponent ], imports: [ @@ -487,6 +489,7 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) CopyButtonComponent, TogglePasswordComponent, ProtobufContentComponent, + BranchAutocompleteComponent, PhoneInputComponent ] }) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cfc919721d..0ab99ccd79 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -59,7 +59,9 @@ "read-more": "Read more", "hide": "Hide", "done": "Done", - "print": "Print" + "print": "Print", + "restore": "Restore", + "confirm": "Confirm" }, "aggregation": { "aggregation": "Aggregation", @@ -322,6 +324,30 @@ "queue-submit-strategy": "Submit strategy", "queue-processing-strategy": "Processing strategy", "queue-configuration": "Queue configuration", + "repository-settings": "Repository settings", + "repository-url": "Repository URL", + "repository-url-required": "Repository URL is required.", + "default-branch": "Default branch name", + "authentication-settings": "Authentication settings", + "auth-method": "Authentication method", + "auth-method-username-password": "Password / access token", + "auth-method-private-key": "Private key", + "password-access-token": "Password / access token", + "change-password-access-token": "Change password / access token", + "private-key": "Private key", + "drop-private-key-file-or": "Drag and drop a private key file or", + "passphrase": "Passphrase", + "enter-passphrase": "Enter passphrase", + "change-passphrase": "Change passphrase", + "check-access": "Check access", + "check-repository-access-success": "Repository access successfully verified!", + "delete-repository-settings-title": "Are you sure you want to delete repository settings?", + "delete-repository-settings-text": "Be careful, after the confirmation the repository settings will be removed and version control feature will be unavailable.", + "auto-commit-settings": "Auto-commit settings", + "auto-commit-entities": "Auto-commit entities", + "no-auto-commit-entities-prompt": "No entities configured for auto-commit", + "delete-auto-commit-settings-title": "Are you sure you want to delete auto-commit settings?", + "delete-auto-commit-settings-text": "Be careful, after the confirmation the auto-commit settings will be removed and auto-commit will be disabled for all entities.", "2fa": { "2fa": "Two-factor authentication", "available-providers": "Available providers", @@ -1819,6 +1845,9 @@ "type-current-tenant": "Current Tenant", "type-current-user": "Current User", "type-current-user-owner": "Current User Owner", + "type-widgets-bundle": "Widgets bundle", + "type-widgets-bundles": "Widgets bundles", + "list-of-widgets-bundles": "{ count, plural, 1 {One widgets bundle} other {List of # widget bundles} }", "search": "Search entities", "selected-entities": "{ count, plural, 1 {1 entity} other {# entities} } selected", "entity-name": "Entity name", @@ -3071,6 +3100,8 @@ "transport-device-msg-rate-limit": "Transport device messages rate limit.", "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", + "tenant-entity-export-rate-limit": "Entity version creation rate limit", + "tenant-entity-import-rate-limit": "Entity version load rate limit", "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", "max-transport-messages-required": "Maximum number of transport messages is required.", "max-transport-messages-range": "Maximum number of transport messages can't be negative", @@ -3253,6 +3284,68 @@ "json-value-invalid": "JSON value has an invalid format", "json-value-required": "JSON value is required." }, + "version-control": { + "version-control": "Version control", + "management": "Version control management", + "branch": "Branch", + "default": "Default", + "select-branch": "Select branch", + "branch-required": "Branch is required", + "create-entity-version": "Create entity version", + "version-name": "Version name", + "version-name-required": "Version name is required", + "author": "Author", + "export-relations": "Export relations", + "export-attributes": "Export attributes", + "export-credentials": "Export credentials", + "entity-versions": "Entity versions", + "versions": "Versions", + "created-time": "Created time", + "version-id": "Version ID", + "no-entity-versions-text": "No entity versions found", + "no-versions-text": "No versions found", + "copy-full-version-id": "Copy full version id", + "create-version": "Create version", + "nothing-to-commit": "No changes to commit", + "restore-version": "Restore version", + "restore-entity-from-version": "Restore entity from version '{{versionName}}'", + "load-relations": "Load relations", + "load-attributes": "Load attributes", + "load-credentials": "Load credentials", + "show-version-diff": "Show version diff", + "diff-entity-with-version": "Diff with entity version '{{versionName}}'", + "previous-difference": "Previous Difference", + "next-difference": "Next Difference", + "current": "Current", + "differences": "{ count, plural, 1 {1 difference} other {# differences} }", + "create-entities-version": "Create entities version", + "default-sync-strategy": "Default sync strategy", + "sync-strategy-merge": "Merge", + "sync-strategy-overwrite": "Overwrite", + "entities-to-export": "Entities to export", + "entities-to-restore": "Entities to restore", + "sync-strategy": "Sync strategy", + "all-entities": "All entities", + "no-entities-to-export-prompt": "Please specify entities to export", + "no-entities-to-restore-prompt": "Please specify entities to restore", + "add-entity-type": "Add entity type", + "remove-all": "Remove all", + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.
{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.
{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "remove-other-entities": "Remove other entities", + "find-existing-entity-by-name": "Find existing entity by name", + "restore-entities-from-version": "Restore entities from version '{{versionName}}'", + "no-entities-restored": "No entities restored", + "created": "{{created}} created", + "updated": "{{updated}} updated", + "deleted": "{{deleted}} deleted", + "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm.", + "auto-commit-to-branch": "auto-commit to {{ branch }} branch", + "default-create-entity-version-name": "{{entityName}} update", + "sync-strategy-merge-hint": "Creates or updates selected entities in the repository. All other repository entities are not modified.", + "sync-strategy-overwrite-hint": "Creates or updates selected entities in the repository. All other repository entities are deleted.", + "device-credentials-conflict": "Failed to load the device with external id {{entityId}}
due to the same credentials are already present in the database for another device.
Please consider disabling the load credentials setting in the restore form.", + "missing-referenced-entity": "Failed to load the {{sourceEntityTypeName}} with external id {{sourceEntityId}}
because it references missing {{targetEntityTypeName}} with id {{targetEntityId}}." + }, "widget": { "widget-library": "Widgets Library", "widget-bundle": "Widgets Bundle", diff --git a/ui-ngx/yarn.lock b/ui-ngx/yarn.lock index 9d0ef325b7..9005658ea8 100644 --- a/ui-ngx/yarn.lock +++ b/ui-ngx/yarn.lock @@ -1751,6 +1751,11 @@ "@turf/helpers" "^6.5.0" "@turf/meta" "^6.5.0" +"@types/ace-diff@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@types/ace-diff/-/ace-diff-2.1.1.tgz#1c08919aae8f9c429fcb139dc564c89dd093cbee" + integrity sha512-O27fCo2Y0njNslOFSewyRhTyXfLhVhleEU5aTI6ZqFTKENJ8L/LA+Y+ZfcHsHTtwrTWjBXqORmqEHH6Qytqw1w== + "@types/canvas-gauges@^2.1.4": version "2.1.4" resolved "https://registry.yarnpkg.com/@types/canvas-gauges/-/canvas-gauges-2.1.4.tgz#063881264597d098e78cf5ad921e8ed20ae2ad16" @@ -2190,6 +2195,13 @@ ace-builds@^1.4.13: resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.4.13.tgz#186f42d3849ebcc6a48b93088a058489897514c1" integrity sha512-SOLzdaQkY6ecPKYRDDg+MY1WoGgXA34cIvYJNNoBMGGUswHmlauU2Hy0UL96vW0Fs/LgFbMUjD+6vqzWTldIYQ== +ace-diff@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/ace-diff/-/ace-diff-3.0.3.tgz#84f685ff3d0b1910539fc39259ac73d8b6581e28" + integrity sha512-CJaV9Oi6BWLWGL2Kj//h5BNXlRCRu1GYHPOT7o+ZSAuJv9PaL9FWr/cCf16IuSVDo7oj6VriO+qgoHR8G9McLA== + dependencies: + diff-match-patch "^1.0.5" + acorn-import-assertions@^1.7.6: version "1.8.0" resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9"