diff --git a/application/pom.xml b/application/pom.xml index 2d6dda0bbe..6d1e009d00 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -69,6 +69,10 @@ org.thingsboard.common cluster-api + + org.thingsboard.common + version-control + org.thingsboard.rule-engine rule-engine-components diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 898128c1ff..6d2c95ee84 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/application/src/main/data/upgrade/3.3.4/schema_update.sql b/application/src/main/data/upgrade/3.3.4/schema_update.sql index 7584d2f732..69df2afdc9 100644 --- a/application/src/main/data/upgrade/3.3.4/schema_update.sql +++ b/application/src/main/data/upgrade/3.3.4/schema_update.sql @@ -14,6 +14,40 @@ -- limitations under the License. -- +ALTER TABLE device + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE device_profile + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE asset + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_chain + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE rule_node + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE dashboard + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE customer + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE widgets_bundle + ADD COLUMN IF NOT EXISTS external_id UUID; +ALTER TABLE entity_view + ADD COLUMN IF NOT EXISTS external_id UUID; + +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_rule_node_external_id ON rule_node(rule_chain_id, external_id); +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); +CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_node_type ON rule_node(type); + +ALTER TABLE admin_settings + ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080'; + CREATE TABLE IF NOT EXISTS queue ( id uuid NOT NULL CONSTRAINT queue_pkey PRIMARY KEY, created_time bigint NOT NULL, @@ -35,3 +69,4 @@ CREATE TABLE IF NOT EXISTS user_auth_settings ( user_id uuid UNIQUE NOT NULL CONSTRAINT fk_user_auth_settings_user_id REFERENCES tb_user(id), two_fa_settings varchar ); + diff --git a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java index efc1e55701..e90ed98351 100644 --- a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java +++ b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java @@ -32,7 +32,9 @@ import java.util.Arrays; "org.thingsboard.server.dao", "org.thingsboard.server.common.stats", "org.thingsboard.server.common.transport.config.ssl", - "org.thingsboard.server.cache"}) + "org.thingsboard.server.cache", + "org.thingsboard.server.springfox" +}) public class ThingsboardInstallApplication { private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 500ad60524..7e689e58fa 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -48,7 +48,6 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.tools.TbRateLimits; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.audit.AuditLogService; @@ -77,9 +76,11 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.usagestats.TbApiUsageClient; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; +import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.executors.ExternalCallExecutorService; import org.thingsboard.server.service.executors.SharedEventLoopGroupService; @@ -220,6 +221,11 @@ public class ActorSystemContext { @Getter private EntityViewService entityViewService; + @Lazy + @Autowired(required = false) + @Getter + private TbEntityViewService tbEntityViewService; + @Autowired @Getter private TelemetrySubscriptionService tsSubService; diff --git a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java index 1415ff4ece..86675a79a8 100644 --- a/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java +++ b/application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java @@ -15,9 +15,11 @@ */ package org.thingsboard.server.config; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; +import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Component; @@ -41,6 +43,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +@Slf4j @Component public class RateLimitProcessingFilter extends GenericFilterBean { @@ -58,7 +61,13 @@ public class RateLimitProcessingFilter extends GenericFilterBean { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { SecurityUser user = getCurrentUser(); if (user != null && !user.isSystemAdmin()) { - var profileConfiguration = tenantProfileCache.get(user.getTenantId()).getDefaultProfileConfiguration(); + var profile = tenantProfileCache.get(user.getTenantId()); + if (profile == null) { + log.debug("[{}] Failed to lookup tenant profile", user.getTenantId()); + errorResponseHandler.handle(new BadCredentialsException("Failed to lookup tenant profile"), (HttpServletResponse) response); + return; + } + var profileConfiguration = profile.getDefaultProfileConfiguration(); if (!checkRateLimits(user.getTenantId(), profileConfiguration.getTenantServerRestLimitsConfiguration(), perTenantLimits, response)) { return; } diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 823bbf35e3..76c631bddb 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -181,13 +181,18 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt @Autowired private OAuth2AuthorizationRequestResolver oAuth2AuthorizationRequestResolver; + @Override + public void configure(WebSecurity web) throws Exception { + web.ignoring().antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**"); + } + @Override protected void configure(HttpSecurity http) throws Exception { - http.authorizeHttpRequests((authorizeHttpRequests) -> - authorizeHttpRequests - .antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**") - .permitAll() - ); +// http.authorizeHttpRequests((authorizeHttpRequests) -> +// authorizeHttpRequests +// .antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**") +// .permitAll() +// ); http.headers().cacheControl().and().frameOptions().disable() .and() .cors() diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index e5934cbf26..54cbdd9b0f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -16,16 +16,16 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.server.common.data.AdminSettings; @@ -34,14 +34,18 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.update.UpdateService; -import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.*; @RestController @TbCoreComponent @@ -60,6 +64,12 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Autowired + private EntitiesVersionControlService versionControlService; + + @Autowired + private TbAutoCommitSettingsService autoCommitSettingsService; + @Autowired private UpdateService updateService; @@ -96,6 +106,7 @@ public class AdminController extends BaseController { @RequestBody AdminSettings adminSettings) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + adminSettings.setTenantId(getTenantId()); adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings)); if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); @@ -180,6 +191,137 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get repository settings (getRepositorySettings)", + notes = "Get the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/repositorySettings") + @ResponseBody + public RepositorySettings getRepositorySettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId())); + versionControlSettings.setPassword(null); + versionControlSettings.setPrivateKey(null); + versionControlSettings.setPrivateKeyPassword(null); + return versionControlSettings; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check repository settings exists (repositorySettingsExists)", + notes = "Check whether the repository settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/repositorySettings/exists") + @ResponseBody + public Boolean repositorySettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return versionControlService.getVersionControlSettings(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the repository settings (saveRepositorySettings)", + notes = "Creates or Updates the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/repositorySettings") + public DeferredResult saveRepositorySettings(@RequestBody RepositorySettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); + ListenableFuture future = versionControlService.saveVersionControlSettings(getTenantId(), settings); + return wrapFuture(Futures.transform(future, savedSettings -> { + savedSettings.setPassword(null); + savedSettings.setPrivateKey(null); + savedSettings.setPrivateKeyPassword(null); + return savedSettings; + }, MoreExecutors.directExecutor())); + } + + @ApiOperation(value = "Delete repository settings (deleteRepositorySettings)", + notes = "Deletes the repository settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public DeferredResult deleteRepositorySettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.DELETE); + return wrapFuture(versionControlService.deleteVersionControlSettings(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + + @ApiOperation(value = "Check repository access (checkRepositoryAccess)", + notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/repositorySettings/checkAccess", method = RequestMethod.POST) + public DeferredResult checkRepositoryAccess( + @ApiParam(value = "A JSON value representing the Repository Settings.") + @RequestBody RepositorySettings settings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + settings = checkNotNull(settings); + return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings)); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)", + notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/autoCommitSettings") + @ResponseBody + public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return checkNotNull(autoCommitSettingsService.get(getTenantId())); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)", + notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @GetMapping("/autoCommitSettings/exists") + @ResponseBody + public Boolean autoCommitSettingsExists() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return autoCommitSettingsService.get(getTenantId()) != null; + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)", + notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @PostMapping("/autoCommitSettings") + public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); + return autoCommitSettingsService.save(getTenantId(), settings); + } + + @ApiOperation(value = "Delete auto commit settings (deleteAutoCommitSettings)", + notes = "Deletes the auto commit settings." + + TENANT_AUTHORITY_PARAGRAPH) + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/autoCommitSettings", method = RequestMethod.DELETE) + @ResponseStatus(value = HttpStatus.OK) + public void deleteAutoCommitSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.DELETE); + autoCommitSettingsService.delete(getTenantId()); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Check for new Platform Releases (checkUpdates)", notes = "Check notifications about new platform releases. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java index f53eb28e20..2c171e8ca6 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AlarmController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AlarmController.java @@ -222,11 +222,7 @@ public class AlarmController extends BaseController { TimePageLink pageLink = createTimePageLink(pageSize, page, textSearch, sortProperty, sortOrder, startTime, endTime); try { - if (getCurrentUser().isCustomerUser()) { - return checkNotNull(alarmService.findCustomerAlarms(getCurrentUser().getTenantId(), getCurrentUser().getCustomerId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); - } else { - return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); - } + return checkNotNull(alarmService.findAlarms(getCurrentUser().getTenantId(), new AlarmQuery(entityId, pageLink, alarmSearchStatus, alarmStatus, fetchOriginator)).get()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 77df4100dc..13ef646eaa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -51,9 +51,9 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.asset.TbAssetService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -143,7 +143,7 @@ public class AssetController extends BaseController { @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/asset", method = RequestMethod.POST) @ResponseBody - public Asset saveAsset(@ApiParam(value = "A JSON value representing the asset.") @RequestBody Asset asset) throws ThingsboardException { + public Asset saveAsset(@ApiParam(value = "A JSON value representing the asset.") @RequestBody Asset asset) throws Exception { if (TB_SERVICE_QUEUE.equals(asset.getType())) { throw new ThingsboardException("Unable to save asset with type " + TB_SERVICE_QUEUE, ThingsboardErrorCode.BAD_REQUEST_PARAMS); } diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 0f8c40956e..a26de329b3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -17,6 +17,10 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -28,6 +32,7 @@ import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.MethodArgumentNotValidException; import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -134,6 +139,7 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.state.DeviceStateService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.telemetry.AlarmSubscriptionService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -277,6 +283,9 @@ public abstract class BaseController { @Autowired protected QueueService queueService; + @Autowired + protected EntitiesVersionControlService vcService; + @Value("${server.log_controller_error_stack_trace}") @Getter private boolean logControllerErrorStackTrace; @@ -883,15 +892,15 @@ public abstract class BaseController { } protected void sendEntityNotificationMsg(TenantId tenantId, EntityId entityId, EdgeEventActionType action) { - sendNotificationMsgToEdgeService(tenantId, null, entityId, null, null, action); + sendNotificationMsgToEdge(tenantId, null, entityId, null, null, action); } protected void sendEntityAssignToEdgeNotificationMsg(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { - sendNotificationMsgToEdgeService(tenantId, edgeId, entityId, null, null, action); + sendNotificationMsgToEdge(tenantId, edgeId, entityId, null, null, action); } - private void sendNotificationMsgToEdgeService(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action) { - tbClusterService.sendNotificationMsgToEdgeService(tenantId, edgeId, entityId, body, type, action); + private void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action) { + tbClusterService.sendNotificationMsgToEdge(tenantId, edgeId, entityId, body, type, action); } protected List findRelatedEdgeIds(TenantId tenantId, EntityId entityId) { @@ -926,4 +935,20 @@ public abstract class BaseController { return MediaType.APPLICATION_OCTET_STREAM; } } + + protected DeferredResult wrapFuture(ListenableFuture future) { + final DeferredResult deferredResult = new DeferredResult<>(); + Futures.addCallback(future, new FutureCallback<>() { + @Override + public void onSuccess(T result) { + deferredResult.setResult(result); + } + + @Override + public void onFailure(Throwable t) { + deferredResult.setErrorResult(t); + } + }, MoreExecutors.directExecutor()); + return deferredResult; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index a26493000a..6a616671eb 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -30,7 +30,7 @@ public class ControllerConstants { protected static final String PAGE_DATA_PARAMETERS = "You can specify parameters to filter the results. " + "The result is wrapped with PageData object that allows you to iterate over result set using pagination. " + "See the 'Model' tab of the Response Class for more details. "; - protected static final String DASHBOARD_ID_PARAM_DESCRIPTION = "A string value representing the device id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; + protected static final String DASHBOARD_ID_PARAM_DESCRIPTION = "A string value representing the dashboard id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String RPC_ID_PARAM_DESCRIPTION = "A string value representing the rpc id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String DEVICE_ID_PARAM_DESCRIPTION = "A string value representing the device id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String ENTITY_VIEW_ID_PARAM_DESCRIPTION = "A string value representing the entity view id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; @@ -49,6 +49,7 @@ public class ControllerConstants { protected static final String RULE_NODE_ID_PARAM_DESCRIPTION = "A string value representing the rule node id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String WIDGET_BUNDLE_ID_PARAM_DESCRIPTION = "A string value representing the widget bundle id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String WIDGET_TYPE_ID_PARAM_DESCRIPTION = "A string value representing the widget type id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; + protected static final String VC_REQUEST_ID_PARAM_DESCRIPTION = "A string value representing the version control request id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String RESOURCE_ID_PARAM_DESCRIPTION = "A string value representing the resource id. For example, '784f394c-42b6-435a-983c-b7beff2784f9'"; protected static final String SYSTEM_AUTHORITY_PARAGRAPH = "\n\nAvailable for users with 'SYS_ADMIN' authority."; protected static final String SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH = "\n\nAvailable for users with 'SYS_ADMIN' or 'TENANT_ADMIN' authority."; @@ -135,6 +136,9 @@ public class ControllerConstants { protected static final String EDGE_ASSIGN_ASYNC_FIRST_STEP_DESCRIPTION = "Assignment works in async way - first, notification event pushed to edge service queue on platform. "; protected static final String EDGE_ASSIGN_RECEIVE_STEP_DESCRIPTION = "(Edge will receive this instantly, if it's currently connected, or once it's going to be connected to platform). "; + protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name."; + protected static final String VERSION_ID_PARAM_DESCRIPTION = "Version id, for example fd82625bdd7d6131cf8027b44ee967012ecaf990. Represents commit hash."; + protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n"; protected static final String MARKDOWN_CODE_BLOCK_END = "\n```"; protected static final String EVENT_ERROR_FILTER_OBJ = MARKDOWN_CODE_BLOCK_START + diff --git a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java index 6dba5584df..2510974991 100644 --- a/application/src/main/java/org/thingsboard/server/controller/CustomerController.java +++ b/application/src/main/java/org/thingsboard/server/controller/CustomerController.java @@ -142,7 +142,7 @@ public class CustomerController extends BaseController { @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/customer", method = RequestMethod.POST) @ResponseBody - public Customer saveCustomer(@ApiParam(value = "A JSON value representing the customer.") @RequestBody Customer customer) throws ThingsboardException { + public Customer saveCustomer(@ApiParam(value = "A JSON value representing the customer.") @RequestBody Customer customer) throws Exception { customer.setTenantId(getTenantId()); checkEntity(customer.getId(), customer, Resource.CUSTOMER); return tbCustomerService.save(customer, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java index 05ef42017f..0530f59c97 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DashboardController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DashboardController.java @@ -180,7 +180,7 @@ public class DashboardController extends BaseController { @ResponseBody public Dashboard saveDashboard( @ApiParam(value = "A JSON value representing the dashboard.") - @RequestBody Dashboard dashboard) throws ThingsboardException { + @RequestBody Dashboard dashboard) throws Exception { dashboard.setTenantId(getTenantId()); checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD); return tbDashboardService.save(dashboard, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 86b4e3ebef..183b5ea048 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -66,9 +66,9 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.device.TbDeviceService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -165,7 +165,7 @@ public class DeviceController extends BaseController { @ResponseBody public Device saveDevice(@ApiParam(value = "A JSON value representing the device.") @RequestBody Device device, @ApiParam(value = "Optional value of the device credentials to be used during device creation. " + - "If omitted, access token will be auto-generated.") @RequestParam(name = "accessToken", required = false) String accessToken) throws ThingsboardException { + "If omitted, access token will be auto-generated.") @RequestParam(name = "accessToken", required = false) String accessToken) throws Exception { device.setTenantId(getCurrentUser().getTenantId()); Device oldDevice = null; if (device.getId() != null) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java index 72d9b1c654..08568a28ce 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceProfileController.java @@ -108,7 +108,7 @@ public class DeviceProfileController extends BaseController { checkParameter(DEVICE_PROFILE_ID, strDeviceProfileId); try { DeviceProfileId deviceProfileId = new DeviceProfileId(toUUID(strDeviceProfileId)); - return checkNotNull(deviceProfileService.findDeviceProfileInfoById(getTenantId(), deviceProfileId)); + return new DeviceProfileInfo(checkDeviceProfileId(deviceProfileId, Operation.READ)); } catch (Exception e) { throw handleException(e); } @@ -199,7 +199,7 @@ public class DeviceProfileController extends BaseController { @ResponseBody public DeviceProfile saveDeviceProfile( @ApiParam(value = "A JSON value representing the device profile.") - @RequestBody DeviceProfile deviceProfile) throws ThingsboardException { + @RequestBody DeviceProfile deviceProfile) throws Exception { deviceProfile.setTenantId(getTenantId()); checkEntity(deviceProfile.getId(), deviceProfile, Resource.DEVICE_PROFILE); return tbDeviceProfileService.save(deviceProfile, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 9cecf86885..b3c1792b6a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -32,6 +32,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.rule.engine.flow.TbRuleChainInputNode; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.edge.Edge; @@ -52,8 +53,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.importing.BulkImportRequest; -import org.thingsboard.server.service.importing.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; @@ -557,7 +558,7 @@ public class EdgeController extends BaseController { edgeId = checkNotNull(edgeId); SecurityUser user = getCurrentUser(); TenantId tenantId = user.getTenantId(); - return edgeService.findMissingToRelatedRuleChains(tenantId, edgeId); + return edgeService.findMissingToRelatedRuleChains(tenantId, edgeId, TbRuleChainInputNode.class.getName()); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java new file mode 100644 index 0000000000..dfefce4b12 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -0,0 +1,508 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import lombok.RequiredArgsConstructor; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.permission.Operation; +import org.thingsboard.server.service.security.permission.Resource; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; +import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; +import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; +import static org.thingsboard.server.controller.ControllerConstants.VC_REQUEST_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.VERSION_ID_PARAM_DESCRIPTION; + +@RestController +@TbCoreComponent +@RequestMapping("/api/entities/vc") +@PreAuthorize("hasAuthority('TENANT_ADMIN')") +@RequiredArgsConstructor +public class EntitiesVersionControlController extends BaseController { + + private final EntitiesVersionControlService versionControlService; + + + @ApiOperation(value = "Save entities version (saveEntitiesVersion)", notes = "" + + "Creates a new version of entities (or a single entity) by request.\n" + + "Supported entity types: CUSTOMER, ASSET, RULE_CHAIN, DASHBOARD, DEVICE_PROFILE, DEVICE, ENTITY_VIEW, WIDGETS_BUNDLE." + NEW_LINE + + "There are two available types of request: `SINGLE_ENTITY` and `COMPLEX`. " + + "Each of them contains version name (`versionName`) and name of a branch (`branch`) to create version (commit) in. " + + "If specified branch does not exists in a remote repo, then new empty branch will be created. " + + "Request of the `SINGLE_ENTITY` type has id of an entity (`entityId`) and additional configuration (`config`) " + + "which has following options: \n" + + "- `saveRelations` - whether to add inbound and outbound relations of type COMMON to created entity version;\n" + + "- `saveAttributes` - to save attributes of server scope (and also shared scope for devices);\n" + + "- `saveCredentials` - when saving a version of a device, to add its credentials to the version." + NEW_LINE + + "An example of a `SINGLE_ENTITY` version create request:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + "\n" + + " \"versionName\": \"Version 1.0\",\n" + + " \"branch\": \"dev\",\n" + + "\n" + + " \"entityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"saveRelations\": true,\n" + + " \"saveAttributes\": true,\n" + + " \"saveCredentials\": false\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "Second request type (`COMPLEX`), additionally to `branch` and `versionName`, contains following properties:\n" + + "- `entityTypes` - a structure with entity types to export and configuration for each entity type; " + + " this configuration has all the options available for `SINGLE_ENTITY` and additionally has these ones: \n" + + " - `allEntities` and `entityIds` - if you want to save the version of all entities of the entity type " + + " then set `allEntities` param to true, otherwise set it to false and specify the list of specific entities (`entityIds`);\n" + + " - `syncStrategy` - synchronization strategy to use for this entity type: when set to `OVERWRITE` " + + " then the list of remote entities of this type will be overwritten by newly added entities. If set to " + + " `MERGE` - existing remote entities of this entity type will not be removed, new entities will just " + + " be added on top (or existing remote entities will be updated).\n" + + "- `syncStrategy` - default synchronization strategy to use when it is not specified for an entity type." + NEW_LINE + + "Example for this type of request:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"type\": \"COMPLEX\",\n" + + "\n" + + " \"versionName\": \"Devices and profiles: release 2\",\n" + + " \"branch\": \"master\",\n" + + "\n" + + " \"syncStrategy\": \"OVERWRITE\",\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"syncStrategy\": null,\n" + + " \"allEntities\": true,\n" + + " \"saveRelations\": true,\n" + + " \"saveAttributes\": true,\n" + + " \"saveCredentials\": true\n" + + " },\n" + + " \"DEVICE_PROFILE\": {\n" + + " \"syncStrategy\": \"MERGE\",\n" + + " \"allEntities\": false,\n" + + " \"entityIds\": [\n" + + " \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " ],\n" + + " \"saveRelations\": true\n" + + " }\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "Response wil contain generated request UUID, that can be then used to retrieve " + + "status of operation via `getVersionCreateRequestStatus`.\n" + + TENANT_AUTHORITY_PARAGRAPH) + @PostMapping("/version") + public DeferredResult saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws Exception { + SecurityUser user = getCurrentUser(); + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE); + return wrapFuture(versionControlService.saveEntitiesVersion(user, request)); + } + + @ApiOperation(value = "Get version create request status (getVersionCreateRequestStatus)", notes = "" + + "Returns the status of previously made version create request. " + NEW_LINE + + "This status contains following properties:\n" + + "- `done` - whether request processing is finished;\n" + + "- `version` - created version info: timestamp, version id (commit hash), commit name and commit author;\n" + + "- `added` - count of items that were created in the remote repo;\n" + + "- `modified` - modified items count;\n" + + "- `removed` - removed items count;\n" + + "- `error` - error message, if an error occurred while handling the request." + NEW_LINE + + "An example of successful status:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"done\": true,\n" + + " \"added\": 10,\n" + + " \"modified\": 2,\n" + + " \"removed\": 5,\n" + + " \"version\": {\n" + + " \"timestamp\": 1655198528000,\n" + + " \"id\":\"8a834dd389ed80e0759ba8ee338b3f1fd160a114\",\n" + + " \"name\": \"My devices v2.0\",\n" + + " \"author\": \"John Doe\"\n" + + " },\n" + + " \"error\": null\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/version/{requestId}/status") + public VersionCreationResult getVersionCreateRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true) + @PathVariable UUID requestId) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return versionControlService.getVersionCreateStatus(getCurrentUser(), requestId); + } + + @ApiOperation(value = "List entity versions (listEntityVersions)", notes = "" + + "Returns list of versions for a specific entity in a concrete branch. \n" + + "You need to specify external id of an entity to list versions for. This is `externalId` property of an entity, " + + "or otherwise if not set - simply id of this entity. \n" + + "If specified branch does not exist - empty page data will be returned. " + NEW_LINE + + "Each version info item has timestamp, id, name and author. Version id can then be used to restore the version. " + + PAGE_DATA_PARAMETERS + NEW_LINE + + "Response example: \n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"data\": [\n" + + " {\n" + + " \"timestamp\": 1655198593000,\n" + + " \"id\": \"fd82625bdd7d6131cf8027b44ee967012ecaf990\",\n" + + " \"name\": \"Devices and assets - v2.0\",\n" + + " \"author\": \"John Doe \"\n" + + " },\n" + + " {\n" + + " \"timestamp\": 1655198528000,\n" + + " \"id\": \"682adcffa9c8a2f863af6f00c4850323acbd4219\",\n" + + " \"name\": \"Update my device\",\n" + + " \"author\": \"John Doe \"\n" + + " },\n" + + " {\n" + + " \"timestamp\": 1655198280000,\n" + + " \"id\": \"d2a6087c2b30e18cc55e7cdda345a8d0dfb959a4\",\n" + + " \"name\": \"Devices and assets - v1.0\",\n" + + " \"author\": \"John Doe \"\n" + + " }\n" + + " ],\n" + + " \"totalPages\": 1,\n" + + " \"totalElements\": 3,\n" + + " \"hasNext\": false\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"}) + public DeferredResult> listEntityVersions(@PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid, + @RequestParam String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink)); + } + + @ApiOperation(value = "List entity type versions (listEntityTypeVersions)", notes = "" + + "Returns list of versions of an entity type in a branch. This is a collected list of versions that were created " + + "for entities of this type in a remote branch. \n" + + "If specified branch does not exist - empty page data will be returned. " + + "The response structure is the same as for `listEntityVersions` API method." + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"}) + public DeferredResult> listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink)); + } + + @ApiOperation(value = "List all versions (listVersions)", notes = "" + + "Lists all available versions in a branch for all entity types. \n" + + "If specified branch does not exist - empty page data will be returned. " + + "The response format is the same as for `listEntityVersions` API method." + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/version", params = {"branch", "pageSize", "page"}) + public DeferredResult> listVersions(@RequestParam String branch, + @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) + @RequestParam int pageSize, + @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) + @RequestParam int page, + @ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION) + @RequestParam(required = false) String textSearch, + @ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp") + @RequestParam(required = false) String sortProperty, + @ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES) + @RequestParam(required = false) String sortOrder) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder); + return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink)); + } + + + @ApiOperation(value = "List entities at version (listEntitiesAtVersion)", notes = "" + + "Returns a list of remote entities of a specific entity type that are available at a concrete version. \n" + + "Each entity item in the result has `externalId` property. " + + "Entities order will be the same as in the repository." + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"}) + public DeferredResult> listEntitiesAtVersion(@PathVariable EntityType entityType, + @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) + @PathVariable String versionId, + @RequestParam String branch) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); + } + + @ApiOperation(value = "List all entities at version (listAllEntitiesAtVersion)", notes = "" + + "Returns a list of all remote entities available in a specific version. " + + "Response type is the same as for listAllEntitiesAtVersion API method. \n" + + "Returned entities order will be the same as in the repository." + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/entity/{versionId}", params = {"branch"}) + public DeferredResult> listAllEntitiesAtVersion(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) + @PathVariable String versionId, + @RequestParam String branch) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); + } + + @ApiOperation(value = "Get entity data info (getEntityDataInfo)", notes = "" + + "Retrieves short info about the remote entity by external id at a concrete version. \n" + + "Returned entity data info contains following properties: " + + "`hasRelations` (whether stored entity data contains relations), `hasAttributes` (contains attributes) and " + + "`hasCredentials` (whether stored device data has credentials)." + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}") + public DeferredResult getEntityDataInfo(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) + @PathVariable String versionId, + @PathVariable EntityType entityType, + @PathVariable UUID externalEntityUuid) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); + return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId)); + } + + @ApiOperation(value = "Compare entity data to version (compareEntityDataToVersion)", notes = "" + + "Returns an object with current entity data and the one at a specific version. " + + "Entity data structure is the same as stored in a repository. " + + TENANT_AUTHORITY_PARAGRAPH) + @GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"}) + public DeferredResult compareEntityDataToVersion(@PathVariable EntityType entityType, + @PathVariable UUID internalEntityUuid, + @RequestParam String branch, + @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) + @RequestParam String versionId) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid); + return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId)); + } + + @ApiOperation(value = "Load entities version (loadEntitiesVersion)", notes = "" + + "Loads specific version of remote entities (or single entity) by request. " + + "Supported entity types: CUSTOMER, ASSET, RULE_CHAIN, DASHBOARD, DEVICE_PROFILE, DEVICE, ENTITY_VIEW, WIDGETS_BUNDLE." + NEW_LINE + + "There are multiple types of request. Each of them requires branch name (`branch`) and version id (`versionId`). " + + "Request of type `SINGLE_ENTITY` is needed to restore a concrete version of a specific entity. It contains " + + "id of a remote entity (`externalEntityId`) and additional configuration (`config`):\n" + + "- `loadRelations` - to update relations list (in case `saveRelations` option was enabled during version creation);\n" + + "- `loadAttributes` - to load entity attributes (if `saveAttributes` config option was enabled);\n" + + "- `loadCredentials` - to update device credentials (if `saveCredentials` option was enabled during version creation)." + NEW_LINE + + "An example of such request:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"type\": \"SINGLE_ENTITY\",\n" + + " \n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + " \n" + + " \"externalEntityId\": {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + + " },\n" + + " \"config\": {\n" + + " \"loadRelations\": false,\n" + + " \"loadAttributes\": true,\n" + + " \"loadCredentials\": true\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "Another request type (`ENTITY_TYPE`) is needed to load specific version of the whole entity types. " + + "It contains a structure with entity types to load and configs for each entity type (`entityTypes`). " + + "For each specified entity type, the method will load all remote entities of this type that are present " + + "at the version. A config for each entity type contains the same options as in `SINGLE_ENTITY` request type, and " + + "additionally contains following options:\n" + + "- `removeOtherEntities` - to remove local entities that are not present on the remote - basically to " + + " overwrite local entity type with the remote one;\n" + + "- `findExistingEntityByName` - when you are loading some remote entities that are not yet present at this tenant, " + + " try to find existing entity by name and update it rather than create new." + NEW_LINE + + "Here is an example of the request to completely restore version of the whole device entity type:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"type\": \"ENTITY_TYPE\",\n" + + "\n" + + " \"branch\": \"dev\",\n" + + " \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" + + "\n" + + " \"entityTypes\": {\n" + + " \"DEVICE\": {\n" + + " \"removeOtherEntities\": true,\n" + + " \"findExistingEntityByName\": false,\n" + + " \"loadRelations\": true,\n" + + " \"loadAttributes\": true,\n" + + " \"loadCredentials\": true\n" + + " }\n" + + " }\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + NEW_LINE + + "The response will contain generated request UUID that is to be used to check the status of operation " + + "via `getVersionLoadRequestStatus`." + + TENANT_AUTHORITY_PARAGRAPH) + @PostMapping("/entity") + public UUID loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws Exception { + SecurityUser user = getCurrentUser(); + accessControlService.checkPermission(user, Resource.VERSION_CONTROL, Operation.WRITE); + return versionControlService.loadEntitiesVersion(user, request); + } + + @ApiOperation(value = "Get version load request status (getVersionLoadRequestStatus)", notes = "" + + "Returns the status of previously made version load request. " + + "The structure contains following parameters:\n" + + "- `done` - if the request was successfully processed;\n" + + "- `result` - a list of load results for each entity type:\n" + + " - `created` - created entities count;\n" + + " - `updated` - updated entities count;\n" + + " - `deleted` - removed entities count.\n" + + "- `error` - if an error occurred during processing, error info:\n" + + " - `type` - error type;\n" + + " - `source` - an external id of remote entity;\n" + + " - `target` - if failed to find referenced entity by external id - this external id;\n" + + " - `message` - error message." + NEW_LINE + + "An example of successfully processed request status:\n" + + MARKDOWN_CODE_BLOCK_START + + "{\n" + + " \"done\": true,\n" + + " \"result\": [\n" + + " {\n" + + " \"entityType\": \"DEVICE\",\n" + + " \"created\": 10,\n" + + " \"updated\": 5,\n" + + " \"deleted\": 5\n" + + " },\n" + + " {\n" + + " \"entityType\": \"ASSET\",\n" + + " \"created\": 4,\n" + + " \"updated\": 0,\n" + + " \"deleted\": 8\n" + + " }\n" + + " ]\n" + + "}" + + MARKDOWN_CODE_BLOCK_END + + TENANT_AUTHORITY_PARAGRAPH + ) + @GetMapping(value = "/entity/{requestId}/status") + public VersionLoadResult getVersionLoadRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true) + @PathVariable UUID requestId) throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + return versionControlService.getVersionLoadStatus(getCurrentUser(), requestId); + } + + + @ApiOperation(value = "List branches (listBranches)", notes = "" + + "Lists branches available in the remote repository. \n\n" + + "Response example: \n" + + MARKDOWN_CODE_BLOCK_START + + "[\n" + + " {\n" + + " \"name\": \"master\",\n" + + " \"default\": true\n" + + " },\n" + + " {\n" + + " \"name\": \"dev\",\n" + + " \"default\": false\n" + + " },\n" + + " {\n" + + " \"name\": \"dev-2\",\n" + + " \"default\": false\n" + + " }\n" + + "]" + + MARKDOWN_CODE_BLOCK_END) + @GetMapping("/branches") + public DeferredResult> listBranches() throws Exception { + accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); + final TenantId tenantId = getTenantId(); + ListenableFuture> branches = versionControlService.listBranches(tenantId); + return wrapFuture(Futures.transform(branches, remoteBranches -> { + List infos = new ArrayList<>(); + BranchInfo defaultBranch; + String defaultBranchName = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); + if (StringUtils.isNotEmpty(defaultBranchName)) { + defaultBranch = new BranchInfo(defaultBranchName, true); + } else { + defaultBranch = remoteBranches.stream().filter(BranchInfo::isDefault).findFirst().orElse(null); + } + if (defaultBranch != null) { + infos.add(defaultBranch); + } + infos.addAll(remoteBranches.stream().filter(b -> !b.equals(defaultBranch)) + .map(b -> new BranchInfo(b.getName(), false)).collect(Collectors.toList())); + return infos; + }, MoreExecutors.directExecutor())); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java index 50b82847ac..efc4237b3f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityRelationController.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.entity.relation.TbEntityRelationService; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import java.util.List; @@ -80,8 +81,8 @@ public class EntityRelationController extends BaseController { public void saveRelation(@ApiParam(value = "A JSON value representing the relation.", required = true) @RequestBody EntityRelation relation) throws ThingsboardException { checkNotNull(relation); - checkEntityId(relation.getFrom(), Operation.WRITE); - checkEntityId(relation.getTo(), Operation.WRITE); + checkCanCreateRelation(relation.getFrom()); + checkCanCreateRelation(relation.getTo()); if (relation.getTypeGroup() == null) { relation.setTypeGroup(RelationTypeGroup.COMMON); } @@ -107,8 +108,9 @@ public class EntityRelationController extends BaseController { checkParameter(TO_TYPE, strToType); EntityId fromId = EntityIdFactory.getByTypeAndId(strFromType, strFromId); EntityId toId = EntityIdFactory.getByTypeAndId(strToType, strToId); - checkEntityId(fromId, Operation.WRITE); - checkEntityId(toId, Operation.WRITE); + checkCanCreateRelation(fromId); + checkCanCreateRelation(toId); + RelationTypeGroup relationTypeGroup = parseRelationTypeGroup(strRelationTypeGroup, RelationTypeGroup.COMMON); EntityRelation relation = new EntityRelation(fromId, toId, strRelationType, relationTypeGroup); tbEntityRelationService.delete(getTenantId(), getCurrentUser().getCustomerId(), relation, getCurrentUser()); @@ -340,6 +342,14 @@ public class EntityRelationController extends BaseController { } } + private void checkCanCreateRelation(EntityId entityId) throws ThingsboardException { + SecurityUser currentUser = getCurrentUser(); + var isTenantAdminAndRelateToSelf = currentUser.isTenantAdmin() && currentUser.getTenantId().equals(entityId); + if (!isTenantAdminAndRelateToSelf) { + checkEntityId(entityId, Operation.WRITE); + } + } + private List filterRelationsByReadPermission(List relationsByQuery) { return relationsByQuery.stream().filter(relationByQuery -> { try { diff --git a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java index 2db9386373..38e7f5bb7d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java +++ b/application/src/main/java/org/thingsboard/server/controller/RuleChainController.java @@ -233,7 +233,7 @@ public class RuleChainController extends BaseController { @ResponseBody public RuleChain saveRuleChain( @ApiParam(value = "A JSON value representing the rule chain.") - @RequestBody RuleChain ruleChain) throws ThingsboardException { + @RequestBody RuleChain ruleChain) throws Exception { ruleChain.setTenantId(getCurrentUser().getTenantId()); checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN); return tbRuleChainService.save(ruleChain, getCurrentUser()); @@ -247,7 +247,7 @@ public class RuleChainController extends BaseController { @ResponseBody public RuleChain saveRuleChain( @ApiParam(value = "A JSON value representing the request.") - @RequestBody DefaultRuleChainCreateRequest request) throws ThingsboardException, IOException { + @RequestBody DefaultRuleChainCreateRequest request) throws Exception { checkNotNull(request); checkParameter(request.getName(), "name"); return tbRuleChainService.saveDefaultByName(getTenantId(), request, getCurrentUser()); @@ -277,7 +277,7 @@ public class RuleChainController extends BaseController { @RequestBody RuleChainMetaData ruleChainMetaData, @ApiParam(value = "Update related rule nodes.") @RequestParam(value = "updateRelated", required = false, defaultValue = "true") boolean updateRelated - ) throws ThingsboardException { + ) throws Exception { TenantId tenantId = getTenantId(); if (debugPerTenantEnabled) { ConcurrentMap debugPerTenantLimits = actorContext.getDebugPerTenantLimits(); diff --git a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java index bc0fa376b1..4342a647c5 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TbResourceController.java @@ -146,7 +146,7 @@ public class TbResourceController extends BaseController { @RequestMapping(value = "/resource", method = RequestMethod.POST) @ResponseBody public TbResource saveResource(@ApiParam(value = "A JSON value representing the Resource.") - @RequestBody TbResource resource) throws ThingsboardException { + @RequestBody TbResource resource) throws Exception { resource.setTenantId(getTenantId()); checkEntity(resource.getId(), resource, Resource.TB_RESOURCE); return tbResourceService.save(resource, getCurrentUser()); diff --git a/application/src/main/java/org/thingsboard/server/controller/TenantController.java b/application/src/main/java/org/thingsboard/server/controller/TenantController.java index 320fdc73bc..ef3c141131 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TenantController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TenantController.java @@ -131,7 +131,7 @@ public class TenantController extends BaseController { @RequestMapping(value = "/tenant/{tenantId}", method = RequestMethod.DELETE) @ResponseStatus(value = HttpStatus.OK) public void deleteTenant(@ApiParam(value = TENANT_ID_PARAM_DESCRIPTION) - @PathVariable(TENANT_ID) String strTenantId) throws ThingsboardException { + @PathVariable(TENANT_ID) String strTenantId) throws Exception { checkParameter(TENANT_ID, strTenantId); TenantId tenantId = TenantId.fromUUID(toUUID(strTenantId)); Tenant tenant = checkTenantId(tenantId, Operation.DELETE); diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index 2f9a1b166e..39064c210c 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,13 +22,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -41,7 +42,6 @@ import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cluster.TbClusterService; import java.util.List; import java.util.Map; @@ -211,7 +211,7 @@ public class EntityActionService { } } - public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, + public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, ActionType actionType, Exception e, Object... additionalInfo) { if (customerId == null || customerId.isNullUid()) { customerId = user.getCustomerId(); @@ -222,6 +222,9 @@ public class EntityActionService { auditLogService.logEntityAction(user.getTenantId(), customerId, user.getId(), user.getName(), entityId, entity, actionType, e, additionalInfo); } + public void sendEntityNotificationMsgToEdge(TenantId tenantId, EntityId entityId, EdgeEventActionType action) { + tbClusterService.sendNotificationMsgToEdge(tenantId, null, entityId, null, null, action); + } private T extractParameter(Class clazz, int index, Object... additionalInfo) { T result = null; @@ -266,4 +269,5 @@ public class EntityActionService { entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString()); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java new file mode 100644 index 0000000000..55b23a99cc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.msg.tools.TbRateLimits; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +@Service +@RequiredArgsConstructor +public class DefaultRateLimitService implements RateLimitService { + + private final TbTenantProfileCache tenantProfileCache; + + private final Map> rateLimits = new ConcurrentHashMap<>(); + + @Override + public boolean checkEntityExportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit); + } + + @Override + public boolean checkEntityImportLimit(TenantId tenantId) { + return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit); + } + + private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function rateLimitConfigExtractor) { + String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration() + .map(rateLimitConfigExtractor).orElse(null); + + Map rateLimits = this.rateLimits.get(rateLimitsKey); + if (StringUtils.isEmpty(rateLimitConfig)) { + if (rateLimits != null) { + rateLimits.remove(tenantId); + if (rateLimits.isEmpty()) { + this.rateLimits.remove(rateLimitsKey); + } + } + return true; + } + + if (rateLimits == null) { + rateLimits = new ConcurrentHashMap<>(); + this.rateLimits.put(rateLimitsKey, rateLimits); + } + TbRateLimits rateLimit = rateLimits.get(tenantId); + if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) { + rateLimit = new TbRateLimits(rateLimitConfig); + rateLimits.put(tenantId, rateLimit); + } + + return rateLimit.tryConsume(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java new file mode 100644 index 0000000000..d3d4244ca1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.apiusage; + +import org.thingsboard.server.common.data.id.TenantId; + +public interface RateLimitService { + + boolean checkEntityExportLimit(TenantId tenantId); + + boolean checkEntityImportLimit(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index e9999d2dc1..5eaeec0b73 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -26,9 +26,9 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.asset.TbAssetService; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java index f279a29aac..7a95bf78f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java @@ -161,20 +161,20 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe configurationDescriptor.set("nodeDefinition", node); scannedComponent.setConfigurationDescriptor(configurationDescriptor); scannedComponent.setClazz(clazzName); - log.info("Processing scanned component: {}", scannedComponent); + log.debug("Processing scanned component: {}", scannedComponent); } catch (Exception e) { log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e); throw new RuntimeException(e); } ComponentDescriptor persistedComponent = componentDescriptorService.findByClazz(TenantId.SYS_TENANT_ID, clazzName); if (persistedComponent == null) { - log.info("Persisting new component: {}", scannedComponent); + log.debug("Persisting new component: {}", scannedComponent); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); } else if (scannedComponent.equals(persistedComponent)) { - log.info("Component is already persisted: {}", persistedComponent); + log.debug("Component is already persisted: {}", persistedComponent); scannedComponent = persistedComponent; } else { - log.info("Component {} will be updated to {}", persistedComponent, scannedComponent); + log.debug("Component {} will be updated to {}", persistedComponent, scannedComponent); componentDescriptorService.deleteByClazz(TenantId.SYS_TENANT_ID, persistedComponent.getClazz()); scannedComponent.setId(persistedComponent.getId()); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); @@ -224,7 +224,7 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe @Override public void discoverComponents() { registerRuleNodeComponents(); - log.info("Found following definitions: {}", components.values()); + log.debug("Found following definitions: {}", components.values()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index 25a3a2d156..c04dd5ce15 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,9 +49,9 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.device.TbDeviceService; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Collection; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java b/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java index 6b5e519f80..dc2b9f4089 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/DefaultEdgeNotificationService.java @@ -137,6 +137,7 @@ public class DefaultEdgeNotificationService implements EdgeNotificationService { case ENTITY_VIEW: case DASHBOARD: case RULE_CHAIN: + case OTA_PACKAGE: future = entityProcessor.processEntityNotification(tenantId, edgeNotificationMsg); break; case CUSTOMER: diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index fd55e9b321..404e5af5fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -28,9 +28,9 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.importing.AbstractBulkImportService; -import org.thingsboard.server.service.importing.BulkImportColumnType; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.Map; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index 04f8788503..6863fbc1c0 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -26,6 +26,7 @@ import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.edge.EdgeEventService; import org.thingsboard.server.dao.edge.EdgeService; +import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.user.UserService; @@ -41,6 +42,7 @@ import org.thingsboard.server.service.edge.rpc.processor.DeviceEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.DeviceProfileEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.EntityEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.EntityViewEdgeProcessor; +import org.thingsboard.server.service.edge.rpc.processor.OtaPackageEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.RelationEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.RuleChainEdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.TelemetryEdgeProcessor; @@ -93,6 +95,9 @@ public class EdgeContextComponent { @Autowired private EdgeRequestsService edgeRequestsService; + @Autowired + private OtaPackageService otaPackageService; + @Autowired private AlarmEdgeProcessor alarmProcessor; @@ -138,6 +143,9 @@ public class EdgeContextComponent { @Autowired private AdminSettingsEdgeProcessor adminSettingsProcessor; + @Autowired + private OtaPackageEdgeProcessor otaPackageEdgeProcessor; + @Autowired private EdgeEventStorageSettings edgeEventStorageSettings; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 6c991444b2..1d30d05fea 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -536,6 +536,8 @@ public final class EdgeGrpcSession implements Closeable { return ctx.getWidgetTypeProcessor().processWidgetTypeToEdge(edgeEvent, msgType, action); case ADMIN_SETTINGS: return ctx.getAdminSettingsProcessor().processAdminSettingsToEdge(edgeEvent); + case OTA_PACKAGE: + return ctx.getOtaPackageEdgeProcessor().processOtaPackageToEdge(edgeEvent, msgType, action); default: log.warn("Unsupported edge event type [{}]", edgeEvent); return null; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index a83609fa8f..c880916e35 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -25,6 +25,7 @@ import org.thingsboard.server.service.edge.rpc.fetch.CustomerUsersEdgeEventFetch import org.thingsboard.server.service.edge.rpc.fetch.DashboardsEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.DeviceProfilesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.EdgeEventFetcher; +import org.thingsboard.server.service.edge.rpc.fetch.OtaPackagesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.RuleChainsEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.SystemWidgetsBundlesEdgeEventFetcher; import org.thingsboard.server.service.edge.rpc.fetch.TenantAdminUsersEdgeEventFetcher; @@ -53,6 +54,7 @@ public class EdgeSyncCursor { fetchers.add(new SystemWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new TenantWidgetsBundlesEdgeEventFetcher(ctx.getWidgetsBundleService())); fetchers.add(new DashboardsEdgeEventFetcher(ctx.getDashboardService())); + fetchers.add(new OtaPackagesEdgeEventFetcher(ctx.getOtaPackageService())); } public boolean hasNext() { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java index 77dc121e38..7943ec0595 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java @@ -20,7 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java new file mode 100644 index 0000000000..be38079d1e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.constructor; + +import com.google.protobuf.ByteString; +import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Component +@TbCoreComponent +public class OtaPackageMsgConstructor { + + public OtaPackageUpdateMsg constructOtaPackageUpdatedMsg(UpdateMsgType msgType, OtaPackage otaPackage) { + OtaPackageUpdateMsg.Builder builder = OtaPackageUpdateMsg.newBuilder() + .setMsgType(msgType) + .setIdMSB(otaPackage.getId().getId().getMostSignificantBits()) + .setIdLSB(otaPackage.getId().getId().getLeastSignificantBits()) + .setDeviceProfileIdMSB(otaPackage.getDeviceProfileId().getId().getMostSignificantBits()) + .setDeviceProfileIdLSB(otaPackage.getDeviceProfileId().getId().getLeastSignificantBits()) + .setType(otaPackage.getType().name()) + .setTitle(otaPackage.getTitle()) + .setVersion(otaPackage.getVersion()) + .setTag(otaPackage.getTag()); + + if (otaPackage.getUrl() != null) { + builder.setUrl(otaPackage.getUrl()); + } + if (otaPackage.getAdditionalInfo() != null) { + builder.setAdditionalInfo(JacksonUtil.toString(otaPackage.getAdditionalInfo())); + } + if (otaPackage.getFileName() != null) { + builder.setFileName(otaPackage.getFileName()); + } + if (otaPackage.getContentType() != null) { + builder.setContentType(otaPackage.getContentType()); + } + if (otaPackage.getChecksumAlgorithm() != null) { + builder.setChecksumAlgorithm(otaPackage.getChecksumAlgorithm().name()); + } + if (otaPackage.getChecksum() != null) { + builder.setChecksum(otaPackage.getChecksum()); + } + if (otaPackage.getDataSize() != null) { + builder.setDataSize(otaPackage.getDataSize()); + } + if (otaPackage.getData() != null) { + builder.setData(ByteString.copyFrom(otaPackage.getData().array())); + } + return builder.build(); + } + + public OtaPackageUpdateMsg constructOtaPackageDeleteMsg(OtaPackageId otaPackageId) { + return OtaPackageUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(otaPackageId.getId().getMostSignificantBits()) + .setIdLSB(otaPackageId.getId().getLeastSignificantBits()).build(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/RuleChainMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/RuleChainMsgConstructor.java index 0fc62cfe41..db67ecf822 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/RuleChainMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/RuleChainMsgConstructor.java @@ -21,7 +21,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.flow.TbRuleChainInputNode; import org.thingsboard.rule.engine.flow.TbRuleChainInputNodeConfiguration; +import org.thingsboard.rule.engine.flow.TbRuleChainOutputNode; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.rule.NodeConnectionInfo; import org.thingsboard.server.common.data.rule.RuleChain; @@ -49,9 +51,8 @@ import java.util.stream.Collectors; @TbCoreComponent public class RuleChainMsgConstructor { - private static final ObjectMapper objectMapper = new ObjectMapper(); - private static final String RULE_CHAIN_INPUT_NODE = "org.thingsboard.rule.engine.flow.TbRuleChainInputNode"; - private static final String TB_RULE_CHAIN_OUTPUT_NODE = "org.thingsboard.rule.engine.flow.TbRuleChainOutputNode"; + private static final String RULE_CHAIN_INPUT_NODE = TbRuleChainInputNode.class.getName(); + private static final String TB_RULE_CHAIN_OUTPUT_NODE = TbRuleChainOutputNode.class.getName(); public RuleChainUpdateMsg constructRuleChainUpdatedMsg(RuleChainId edgeRootRuleChainId, UpdateMsgType msgType, RuleChain ruleChain) { RuleChainUpdateMsg.Builder builder = RuleChainUpdateMsg.newBuilder() @@ -210,13 +211,11 @@ public class RuleChainMsgConstructor { private List filterNodes_V_3_3_0(List nodes) { List result = new ArrayList<>(); for (RuleNode node : nodes) { - switch (node.getType()) { - case RULE_CHAIN_INPUT_NODE: - case TB_RULE_CHAIN_OUTPUT_NODE: - log.trace("Skipping not supported rule node {}", node); - break; - default: - result.add(node); + if (RULE_CHAIN_INPUT_NODE.equals(node.getType()) + || TB_RULE_CHAIN_OUTPUT_NODE.equals(node.getType())) { + log.trace("Skipping not supported rule node {}", node); + } else { + result.add(node); } } return result; @@ -280,7 +279,7 @@ public class RuleChainMsgConstructor { .setTargetRuleChainIdMSB(ruleChainConnectionInfo.getTargetRuleChainId().getId().getMostSignificantBits()) .setTargetRuleChainIdLSB(ruleChainConnectionInfo.getTargetRuleChainId().getId().getLeastSignificantBits()) .setType(ruleChainConnectionInfo.getType()) - .setAdditionalInfo(objectMapper.writeValueAsString(additionalInfo)) + .setAdditionalInfo(JacksonUtil.OBJECT_MAPPER.writeValueAsString(additionalInfo)) .build(); } @@ -291,8 +290,8 @@ public class RuleChainMsgConstructor { .setType(node.getType()) .setName(node.getName()) .setDebugMode(node.isDebugMode()) - .setConfiguration(objectMapper.writeValueAsString(node.getConfiguration())) - .setAdditionalInfo(objectMapper.writeValueAsString(node.getAdditionalInfo())) + .setConfiguration(JacksonUtil.OBJECT_MAPPER.writeValueAsString(node.getConfiguration())) + .setAdditionalInfo(JacksonUtil.OBJECT_MAPPER.writeValueAsString(node.getAdditionalInfo())) .build(); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java index 0f4ecf7e30..ee10c7859b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java @@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings))); - AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); + AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings))); @@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates))); - AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); + AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue()); result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS, EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates))); @@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher { } } - private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) { + private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) { AdminSettings tenantMailSettings = new AdminSettings(); + tenantMailSettings.setTenantId(tenantId); jsonValue.put("useSystemMailSettings", true); tenantMailSettings.setJsonValue(jsonValue); tenantMailSettings.setKey(key); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/OtaPackagesEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/OtaPackagesEdgeEventFetcher.java new file mode 100644 index 0000000000..14b8bac3f4 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/OtaPackagesEdgeEventFetcher.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.fetch; + +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.ota.OtaPackageService; + +@AllArgsConstructor +@Slf4j +public class OtaPackagesEdgeEventFetcher extends BasePageableEdgeEventFetcher { + + private final OtaPackageService otaPackageService; + + @Override + PageData fetchPageData(TenantId tenantId, Edge edge, PageLink pageLink) { + return otaPackageService.findTenantOtaPackagesByTenantId(tenantId, pageLink); + } + + @Override + EdgeEvent constructEdgeEvent(TenantId tenantId, Edge edge, OtaPackageInfo otaPackageInfo) { + return EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.OTA_PACKAGE, + EdgeEventActionType.ADDED, otaPackageInfo.getId(), null); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantWidgetsBundlesEdgeEventFetcher.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantWidgetsBundlesEdgeEventFetcher.java index 54a5e202ab..9983e4b0b2 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantWidgetsBundlesEdgeEventFetcher.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/TenantWidgetsBundlesEdgeEventFetcher.java @@ -32,6 +32,6 @@ public class TenantWidgetsBundlesEdgeEventFetcher extends BaseWidgetsBundlesEdge } @Override protected PageData findWidgetsBundles(TenantId tenantId, PageLink pageLink) { - return widgetsBundleService.findAllTenantWidgetsBundlesByTenantIdAndPageLink(tenantId, pageLink); + return widgetsBundleService.findTenantWidgetsBundlesByTenantId(tenantId, pageLink); } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 09bef32af9..8f37d743b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EdgeUtils; @@ -46,12 +47,15 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeEventService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.service.edge.rpc.constructor.AdminSettingsMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AlarmMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.AssetMsgConstructor; @@ -61,6 +65,7 @@ import org.thingsboard.server.service.edge.rpc.constructor.DeviceMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.DeviceProfileMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.EntityDataMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.EntityViewMsgConstructor; +import org.thingsboard.server.service.edge.rpc.constructor.OtaPackageMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.RelationMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.RuleChainMsgConstructor; import org.thingsboard.server.service.edge.rpc.constructor.UserMsgConstructor; @@ -137,6 +142,16 @@ public abstract class BaseEdgeProcessor { @Autowired protected WidgetTypeService widgetTypeService; + @Autowired + protected OtaPackageService otaPackageService; + + @Autowired + protected PartitionService partitionService; + + @Autowired + @Lazy + protected TbQueueProducerProvider producerProvider; + @Autowired protected DataValidator deviceValidator; @@ -182,6 +197,9 @@ public abstract class BaseEdgeProcessor { @Autowired protected AdminSettingsMsgConstructor adminSettingsMsgConstructor; + @Autowired + protected OtaPackageMsgConstructor otaPackageMsgConstructor; + @Autowired protected DbCallbackExecutorService dbCallbackExecutorService; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/OtaPackageEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/OtaPackageEdgeProcessor.java new file mode 100644 index 0000000000..0ccce4c346 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/OtaPackageEdgeProcessor.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; + +@Component +@Slf4j +@TbCoreComponent +public class OtaPackageEdgeProcessor extends BaseEdgeProcessor { + + public DownlinkMsg processOtaPackageToEdge(EdgeEvent edgeEvent, UpdateMsgType msgType, EdgeEventActionType action) { + OtaPackageId otaPackageId = new OtaPackageId(edgeEvent.getEntityId()); + DownlinkMsg downlinkMsg = null; + switch (action) { + case ADDED: + case UPDATED: + OtaPackage otaPackage = otaPackageService.findOtaPackageById(edgeEvent.getTenantId(), otaPackageId); + if (otaPackage != null) { + OtaPackageUpdateMsg otaPackageUpdateMsg = + otaPackageMsgConstructor.constructOtaPackageUpdatedMsg(msgType, otaPackage); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addOtaPackageUpdateMsg(otaPackageUpdateMsg) + .build(); + } + break; + case DELETED: + OtaPackageUpdateMsg otaPackageUpdateMsg = + otaPackageMsgConstructor.constructOtaPackageDeleteMsg(otaPackageId); + downlinkMsg = DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addOtaPackageUpdateMsg(otaPackageUpdateMsg) + .build(); + break; + } + return downlinkMsg; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java index 769d988507..f30111e1ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java @@ -52,6 +52,8 @@ import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.session.SessionMsgType; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.util.JsonUtils; @@ -61,9 +63,12 @@ import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import javax.annotation.Nullable; +import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -77,14 +82,19 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { private final Gson gson = new Gson(); + private TbQueueProducer> tbCoreMsgProducer; + + @PostConstruct + public void init() { + tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer(); + } + public List> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) { log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData); List> result = new ArrayList<>(); EntityId entityId = constructEntityId(entityData); if ((entityData.hasPostAttributesMsg() || entityData.hasPostTelemetryMsg() || entityData.hasAttributesUpdatedMsg()) && entityId != null) { - // @voba - in terms of performance we should not fetch device from DB by id - // TbMsgMetaData metaData = constructBaseMsgMetadata(tenantId, entityId); - TbMsgMetaData metaData = new TbMsgMetaData(); + TbMsgMetaData metaData = constructBaseMsgMetadata(tenantId, entityId); metaData.putValue(DataConstants.MSG_SOURCE_KEY, DataConstants.EDGE_MSG_SOURCE); if (entityData.hasPostAttributesMsg()) { result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData)); @@ -96,6 +106,20 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor { if (entityData.hasPostTelemetryMsg()) { result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData)); } + if (EntityType.DEVICE.equals(entityId.getEntityType())) { + DeviceId deviceId = new DeviceId(entityId.getId()); + + TransportProtos.DeviceActivityProto deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .setLastActivityTime(System.currentTimeMillis()).build(); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId); + tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(deviceId.getId(), + TransportProtos.ToCoreMsg.newBuilder().setDeviceActivityMsg(deviceActivityMsg).build()), null); + } } if (entityData.hasAttributeDeleteMsg()) { result.add(processAttributeDeleteMsg(tenantId, entityId, entityData.getAttributeDeleteMsg(), entityData.getEntityType())); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index 9d87cabb2c..571e6e5aac 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; @@ -60,7 +61,6 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.edge.EdgeEventService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -72,6 +72,7 @@ import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg; import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; +import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; import java.util.ArrayList; @@ -100,8 +101,9 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { @Autowired private DeviceService deviceService; + @Lazy @Autowired - private EntityViewService entityViewService; + private TbEntityViewService entityViewService; @Autowired private DeviceProfileService deviceProfileService; @@ -378,7 +380,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { } List> futures = new ArrayList<>(); for (EntityView entityView : entityViews) { - ListenableFuture future = relationService.checkRelation(tenantId, edge.getId(), entityView.getId(), + ListenableFuture future = relationService.checkRelationAsync(tenantId, edge.getId(), entityView.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE); futures.add(Futures.transformAsync(future, result -> { if (Boolean.TRUE.equals(result)) { @@ -413,11 +415,11 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { } private ListenableFuture saveEdgeEvent(TenantId tenantId, - EdgeId edgeId, - EdgeEventType type, - EdgeEventActionType action, - EntityId entityId, - JsonNode body) { + EdgeId edgeId, + EdgeEventType type, + EdgeEventActionType action, + EntityId entityId, + JsonNode body) { log.trace("Pushing edge event to edge queue. tenantId [{}], edgeId [{}], type [{}], action[{}], entityId [{}], body [{}]", tenantId, edgeId, type, action, entityId, body); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 55f55d54eb..22bf93cd2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -23,6 +23,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; @@ -40,11 +41,13 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; @Slf4j @@ -61,7 +64,7 @@ public abstract class AbstractTbEntityService { @Autowired protected DbCallbackExecutorService dbExecutor; - @Autowired + @Autowired(required = false) protected TbNotificationEntityService notificationEntityService; @Autowired(required = false) protected EdgeService edgeService; @@ -71,6 +74,8 @@ public abstract class AbstractTbEntityService { protected CustomerService customerService; @Autowired protected TbClusterService tbClusterService; + @Autowired(required = false) + private EntitiesVersionControlService vcService; protected ListenableFuture removeAlarmsByEntityId(TenantId tenantId, EntityId entityId) { ListenableFuture> alarmsFuture = @@ -127,4 +132,13 @@ public abstract class AbstractTbEntityService { protected I emptyId(EntityType entityType) { return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID); } + + protected ListenableFuture autoCommit(User user, EntityId entityId) throws Exception { + if (vcService != null) { + return vcService.autoCommit(user, entityId); + } else { + // We do not support auto-commit for rule engine + return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index cbe8e2af11..5f6ca9ce4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -262,7 +262,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS logEntityAction(tenantId, relation.getTo(), null, customerId, actionType, user, additionalInfo); try { if (!relation.getFrom().getEntityType().equals(EntityType.EDGE) && !relation.getTo().getEntityType().equals(EntityType.EDGE)) { - sendNotificationMsgToEdgeService(tenantId, null, null, JacksonUtil.toString(relation), + sendNotificationMsgToEdge(tenantId, null, null, JacksonUtil.toString(relation), EdgeEventType.RELATION, edgeTypeByActionType(actionType)); } } catch (Exception e) { @@ -271,18 +271,18 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS } private void sendEntityNotificationMsg(TenantId tenantId, EntityId entityId, EdgeEventActionType action) { - sendNotificationMsgToEdgeService(tenantId, null, entityId, null, null, action); + sendNotificationMsgToEdge(tenantId, null, entityId, null, null, action); } private void sendEntityAssignToCustomerNotificationMsg(TenantId tenantId, EntityId entityId, CustomerId customerId, EdgeEventActionType action) { try { - sendNotificationMsgToEdgeService(tenantId, null, entityId, JacksonUtil.toString(customerId), null, action); + sendNotificationMsgToEdge(tenantId, null, entityId, JacksonUtil.toString(customerId), null, action); } catch (Exception e) { log.warn("Failed to push assign/unassign to/from customer to core: {}", customerId, e); } } - protected void sendAlarmDeleteNotificationMsg(TenantId tenantId, Alarm alarm, List edgeIds, String body) { + private void sendAlarmDeleteNotificationMsg(TenantId tenantId, Alarm alarm, List edgeIds, String body) { try { sendDeleteNotificationMsg(tenantId, alarm.getId(), edgeIds, body); } catch (Exception e) { @@ -290,8 +290,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS } } - protected void sendDeleteNotificationMsg(TenantId tenantId, I entityId, E entity, - List edgeIds) { + private void sendDeleteNotificationMsg(TenantId tenantId, I entityId, E entity, List edgeIds) { try { sendDeleteNotificationMsg(tenantId, entityId, edgeIds, null); } catch (Exception e) { @@ -302,18 +301,18 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS private void sendDeleteNotificationMsg(TenantId tenantId, EntityId entityId, List edgeIds, String body) { if (edgeIds != null && !edgeIds.isEmpty()) { for (EdgeId edgeId : edgeIds) { - sendNotificationMsgToEdgeService(tenantId, edgeId, entityId, body, null, EdgeEventActionType.DELETED); + sendNotificationMsgToEdge(tenantId, edgeId, entityId, body, null, EdgeEventActionType.DELETED); } } } private void sendEntityAssignToEdgeNotificationMsg(TenantId tenantId, EdgeId edgeId, EntityId entityId, EdgeEventActionType action) { - sendNotificationMsgToEdgeService(tenantId, edgeId, entityId, null, null, action); + sendNotificationMsgToEdge(tenantId, edgeId, entityId, null, null, action); } - private void sendNotificationMsgToEdgeService(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, - EdgeEventType type, EdgeEventActionType action) { - tbClusterService.sendNotificationMsgToEdgeService(tenantId, edgeId, entityId, body, type, action); + private void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, + EdgeEventType type, EdgeEventActionType action) { + tbClusterService.sendNotificationMsgToEdge(tenantId, edgeId, entityId, body, type, action); } private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { @@ -348,14 +347,14 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS return EdgeEventActionType.RELATION_ADD_OR_UPDATE; case RELATION_DELETED: return EdgeEventActionType.RELATION_DELETED; - case ASSIGNED_TO_EDGE: - return EdgeEventActionType.ASSIGNED_TO_EDGE; - case UNASSIGNED_FROM_EDGE: - return EdgeEventActionType.UNASSIGNED_FROM_EDGE; case ASSIGNED_TO_CUSTOMER: return EdgeEventActionType.ASSIGNED_TO_CUSTOMER; case UNASSIGNED_FROM_CUSTOMER: return EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER; + case ASSIGNED_TO_EDGE: + return EdgeEventActionType.ASSIGNED_TO_EDGE; + case UNASSIGNED_FROM_EDGE: + return EdgeEventActionType.UNASSIGNED_FROM_EDGE; default: return null; } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/SimpleTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/SimpleTbEntityService.java index 49b94f4921..7c265a4fa3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/SimpleTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/SimpleTbEntityService.java @@ -16,15 +16,14 @@ package org.thingsboard.server.service.entitiy; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.exception.ThingsboardException; public interface SimpleTbEntityService { - default T save(T entity) throws ThingsboardException { + default T save(T entity) throws Exception { return save(entity, null); } - T save(T entity, User user) throws ThingsboardException; + T save(T entity, User user) throws Exception; void delete(T entity, User user); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index 8b56ed0bc2..d73e384658 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -41,11 +41,12 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb private final AssetService assetService; @Override - public Asset save(Asset asset, User user) throws ThingsboardException { + public Asset save(Asset asset, User user) throws Exception { ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = asset.getTenantId(); try { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); + autoCommit(user, savedAsset.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset, asset.getCustomerId(), actionType, user); return savedAsset; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/TbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/TbAssetService.java index 7c4f337abc..303aabc35b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/TbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/TbAssetService.java @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TbAssetService { - Asset save(Asset asset, User user) throws ThingsboardException; + Asset save(Asset asset, User user) throws Exception; ListenableFuture delete(Asset asset, User user); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java index 2e7ab54a23..07ef6a2eef 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java @@ -21,7 +21,6 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,14 +34,13 @@ import java.util.List; public class DefaultTbCustomerService extends AbstractTbEntityService implements TbCustomerService { @Override - public Customer save(Customer customer, User user) throws ThingsboardException { + public Customer save(Customer customer, User user) throws Exception { ActionType actionType = customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = customer.getTenantId(); - CustomerId customerId = customer.getId(); try { Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, customerId, actionType, user); - + autoCommit(user, savedCustomer.getId()); + notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user); return savedCustomer; } catch (Exception e) { notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.CUSTOMER), customer, actionType, user, e); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java index f815be7f36..dc035798e3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java @@ -45,11 +45,12 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement private final DashboardService dashboardService; @Override - public Dashboard save(Dashboard dashboard, User user) throws ThingsboardException { + public Dashboard save(Dashboard dashboard, User user) throws Exception { ActionType actionType = dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = dashboard.getTenantId(); try { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); + autoCommit(user, savedDashboard.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard, null, actionType, user); return savedDashboard; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index 43c22b53a3..86d187c1f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -58,11 +58,12 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T private final TenantService tenantService; @Override - public Device save(Device device, Device oldDevice, String accessToken, User user) throws ThingsboardException { + public Device save(Device device, Device oldDevice, String accessToken, User user) throws Exception { ActionType actionType = device.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = device.getTenantId(); try { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); + autoCommit(user, savedDevice.getId()); notificationEntityService.notifyCreateOrUpdateDevice(tenantId, savedDevice.getId(), savedDevice.getCustomerId(), savedDevice, oldDevice, actionType, user); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/TbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/TbDeviceService.java index fd51caacf5..3ffc120067 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/TbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/TbDeviceService.java @@ -31,7 +31,7 @@ import org.thingsboard.server.dao.device.claim.ReclaimResult; public interface TbDeviceService { - Device save(Device device, Device oldDevice, String accessToken, User user) throws ThingsboardException; + Device save(Device device, Device oldDevice, String accessToken, User user) throws Exception; Device saveDeviceWithCredentials(Device device, DeviceCredentials deviceCredentials, User user) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/profile/DefaultTbDeviceProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/profile/DefaultTbDeviceProfileService.java index 0d6b870e21..650b35db0d 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/profile/DefaultTbDeviceProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/profile/DefaultTbDeviceProfileService.java @@ -43,7 +43,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple private final OtaPackageStateService otaPackageStateService; @Override - public DeviceProfile save(DeviceProfile deviceProfile, User user) throws ThingsboardException { + public DeviceProfile save(DeviceProfile deviceProfile, User user) throws Exception { ActionType actionType = deviceProfile.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = deviceProfile.getTenantId(); try { @@ -60,7 +60,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple } } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - + autoCommit(user, savedDeviceProfile.getId()); tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedDeviceProfile.getId(), actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java index a49b17ee51..08f241d112 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/DefaultTbEntityViewService.java @@ -23,6 +23,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import org.springframework.util.ConcurrentReferenceHashMap; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; @@ -40,10 +41,11 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -52,13 +54,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.isBlank; @Service -@TbCoreComponent @AllArgsConstructor @Slf4j public class DefaultTbEntityViewService extends AbstractTbEntityService implements TbEntityViewService { @@ -68,23 +71,42 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen private final TelemetrySubscriptionService tsSubService; private final TimeseriesService tsService; + final Map>> localCache = new ConcurrentHashMap<>(); + @Override public EntityView save(EntityView entityView, EntityView existingEntityView, User user) throws ThingsboardException { ActionType actionType = entityView.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = entityView.getTenantId(); + try { + EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView)); + this.updateEntityViewAttributes(tenantId, savedEntityView, existingEntityView, user); + notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView, + null, actionType, user); + localCache.computeIfAbsent(savedEntityView.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>()).clear(); + tbClusterService.broadcastEntityStateChangeEvent(savedEntityView.getTenantId(), savedEntityView.getId(), + entityView.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + return savedEntityView; + } catch (Exception e) { + notificationEntityService.logEntityAction(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, actionType, user, e); + throw e; + } + } + + @Override + public void updateEntityViewAttributes(TenantId tenantId, EntityView savedEntityView, EntityView oldEntityView, User user) throws ThingsboardException { try { List> futures = new ArrayList<>(); - if (existingEntityView != null) { - if (existingEntityView.getKeys() != null && existingEntityView.getKeys().getAttributes() != null) { - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.CLIENT_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SERVER_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); - futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SHARED_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user)); + + if (oldEntityView != null) { + if (oldEntityView.getKeys() != null && oldEntityView.getKeys().getAttributes() != null) { + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.CLIENT_SCOPE, oldEntityView.getKeys().getAttributes().getCs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SERVER_SCOPE, oldEntityView.getKeys().getAttributes().getSs(), user)); + futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SHARED_SCOPE, oldEntityView.getKeys().getAttributes().getSh(), user)); } - List tsKeys = existingEntityView.getKeys() != null && existingEntityView.getKeys().getTimeseries() != null ? - existingEntityView.getKeys().getTimeseries() : Collections.emptyList(); - futures.add(deleteLatestFromEntityView(existingEntityView, tsKeys, user)); + List tsKeys = oldEntityView.getKeys() != null && oldEntityView.getKeys().getTimeseries() != null ? + oldEntityView.getKeys().getTimeseries() : Collections.emptyList(); + futures.add(deleteLatestFromEntityView(oldEntityView, tsKeys, user)); } - EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView)); if (savedEntityView.getKeys() != null) { if (savedEntityView.getKeys().getAttributes() != null) { futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), user)); @@ -100,14 +122,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen throw new RuntimeException("Failed to copy attributes to entity view", e); } } - - notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedEntityView.getId(), savedEntityView, - null, actionType, user); - - return savedEntityView; } catch (Exception e) { - notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.ENTITY_VIEW), entityView, - actionType, user, e); throw e; } } @@ -121,6 +136,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen entityViewService.deleteEntityView(tenantId, entityViewId); notificationEntityService.notifyDeleteEntity(tenantId, entityViewId, entityView, entityView.getCustomerId(), ActionType.DELETED, relatedEdgeIds, user, entityViewId.toString()); + + localCache.computeIfAbsent(tenantId, (k) -> new ConcurrentReferenceHashMap<>()).clear(); + tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityViewId, ComponentLifecycleEvent.DELETED); } catch (Exception e) { notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.ENTITY_VIEW), ActionType.DELETED, user, e, entityViewId.toString()); @@ -208,6 +226,51 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen } } + @Override + public ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId) { + Map> localCacheByTenant = localCache.computeIfAbsent(tenantId, (k) -> new ConcurrentReferenceHashMap<>()); + List fromLocalCache = localCacheByTenant.get(entityId); + if (fromLocalCache != null) { + return Futures.immediateFuture(fromLocalCache); + } + + ListenableFuture> future = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId); + + return Futures.transform(future, (entityViewList) -> { + localCacheByTenant.put(entityId, entityViewList); + return entityViewList; + }, MoreExecutors.directExecutor()); + } + + @Override + public void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg) { + Map> localCacheByTenant = localCache.computeIfAbsent(componentLifecycleMsg.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>()); + EntityViewId entityViewId = new EntityViewId(componentLifecycleMsg.getEntityId().getId()); + deleteOldCacheValue(localCacheByTenant, entityViewId); + if (componentLifecycleMsg.getEvent() != ComponentLifecycleEvent.DELETED) { + EntityView entityView = entityViewService.findEntityViewById(componentLifecycleMsg.getTenantId(), entityViewId); + if (entityView != null) { + localCacheByTenant.remove(entityView.getEntityId()); + } + } + } + + private void deleteOldCacheValue(Map> localCacheByTenant, EntityViewId entityViewId) { + for (var entry : localCacheByTenant.entrySet()) { + EntityView toDelete = null; + for (EntityView view : entry.getValue()) { + if (entityViewId.equals(view.getId())) { + toDelete = view; + break; + } + } + if (toDelete != null) { + entry.getValue().remove(toDelete); + break; + } + } + } + private ListenableFuture> copyAttributesFromEntityToEntityView(EntityView entityView, String scope, Collection keys, User user) throws ThingsboardException { EntityViewId entityId = entityView.getId(); if (keys != null && !keys.isEmpty()) { @@ -223,8 +286,8 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen long lastUpdateTs = attributeKvEntry.getLastUpdateTs(); return startTime == 0 && endTime == 0 || (endTime == 0 && startTime < lastUpdateTs) || - (startTime == 0 && endTime > lastUpdateTs) - ? true : startTime < lastUpdateTs && endTime > lastUpdateTs; + (startTime == 0 && endTime > lastUpdateTs) || + (startTime < lastUpdateTs && endTime > lastUpdateTs); }).collect(Collectors.toList()); tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope, attributes, new FutureCallback() { @Override diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/TbEntityViewService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/TbEntityViewService.java index 7012e39aa6..28ead13853 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/TbEntityViewService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/entityview/TbEntityViewService.java @@ -15,19 +15,26 @@ */ package org.thingsboard.server.service.entitiy.entityview; +import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.plugin.ComponentLifecycleListener; -public interface TbEntityViewService { +import java.util.List; + +public interface TbEntityViewService extends ComponentLifecycleListener { EntityView save(EntityView entityView, EntityView existingEntityView, User user) throws ThingsboardException; + void updateEntityViewAttributes(TenantId tenantId, EntityView savedEntityView, EntityView oldEntityView, User user) throws ThingsboardException; + void delete(EntityView entity, User user) throws ThingsboardException; EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer, User user) throws ThingsboardException; @@ -40,4 +47,6 @@ public interface TbEntityViewService { EntityView unassignEntityViewFromEdge(TenantId tenantId, CustomerId customerId, EntityView entityView, Edge edge, User user) throws ThingsboardException; EntityView unassignEntityViewFromCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer, User user) throws ThingsboardException; + + ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java index 68b115e70a..00cf70db0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java @@ -48,9 +48,12 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen ActionType actionType = saveOtaPackageInfoRequest.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = saveOtaPackageInfoRequest.getTenantId(); try { - OtaPackageInfo savedOtaPackageInfo = checkNotNull(otaPackageService.saveOtaPackageInfo( - new OtaPackageInfo(saveOtaPackageInfoRequest), saveOtaPackageInfoRequest.isUsesUrl())); - notificationEntityService.logEntityAction(tenantId, savedOtaPackageInfo.getId(), savedOtaPackageInfo, actionType, user); + OtaPackageInfo savedOtaPackageInfo = otaPackageService.saveOtaPackageInfo(new OtaPackageInfo(saveOtaPackageInfoRequest), saveOtaPackageInfoRequest.isUsesUrl()); + + boolean sendToEdge = savedOtaPackageInfo.hasUrl() || savedOtaPackageInfo.isHasData(); + notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedOtaPackageInfo.getId(), + savedOtaPackageInfo, user, actionType, sendToEdge, null); + return savedOtaPackageInfo; } catch (Exception e) { notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), saveOtaPackageInfoRequest, @@ -84,7 +87,8 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen otaPackage.setData(ByteBuffer.wrap(data)); otaPackage.setDataSize((long) data.length); OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); - notificationEntityService.logEntityAction(tenantId, savedOtaPackage.getId(), savedOtaPackage, ActionType.UPDATED, user); + notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedOtaPackage.getId(), + savedOtaPackage, user, ActionType.UPDATED, true, null); return savedOtaPackage; } catch (Exception e) { notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), ActionType.UPDATED, @@ -99,8 +103,8 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen OtaPackageId otaPackageId = otaPackageInfo.getId(); try { otaPackageService.deleteOtaPackage(tenantId, otaPackageId); - notificationEntityService.logEntityAction(tenantId, otaPackageId, otaPackageInfo, ActionType.DELETED, user, - otaPackageInfo.getId().toString()); + notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, otaPackageId, otaPackageInfo, + user, ActionType.DELETED, true, null, otaPackageInfo.getId().toString()); } catch (Exception e) { notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), ActionType.DELETED, user, e, otaPackageId.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 50dc591a52..d6dbb97269 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -19,7 +19,6 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; @@ -29,8 +28,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; import org.thingsboard.server.service.entitiy.queue.TbQueueService; import org.thingsboard.server.service.install.InstallScripts; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import java.util.Collections; +import java.util.concurrent.TimeUnit; @Service @TbCoreComponent @@ -42,6 +43,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T private final InstallScripts installScripts; private final TbQueueService tbQueueService; private final TenantProfileService tenantProfileService; + private final EntitiesVersionControlService versionControlService; @Override public Tenant save(Tenant tenant) throws Exception { @@ -64,10 +66,15 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T } @Override - public void delete(Tenant tenant) throws ThingsboardException { - TenantId tenantId = tenant.getId(); - tenantService.deleteTenant(tenantId); - tenantProfileCache.evict(tenantId); - notificationEntityService.notifyDeleteTenant(tenant); + public void delete(Tenant tenant) throws Exception { + try { + TenantId tenantId = tenant.getId(); + tenantService.deleteTenant(tenantId); + tenantProfileCache.evict(tenantId); + notificationEntityService.notifyDeleteTenant(tenant); + versionControlService.deleteVersionControlSettings(tenantId).get(1, TimeUnit.MINUTES); + } catch (Exception e) { + throw e; + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/TbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/TbTenantService.java index 20e910f7b0..2dc8fa452c 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/TbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/TbTenantService.java @@ -16,12 +16,11 @@ package org.thingsboard.server.service.entitiy.tenant; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.exception.ThingsboardException; public interface TbTenantService { Tenant save(Tenant tenant) throws Exception; - void delete(Tenant tenant) throws ThingsboardException; + void delete(Tenant tenant) throws Exception; } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index cece3377b0..14e49e056f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; @@ -162,6 +163,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Getter private boolean persistActivityToTelemetry; + @Lazy @Autowired private QueueService queueService; @@ -265,6 +267,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override public void createAdminSettings() throws Exception { AdminSettings generalSettings = new AdminSettings(); + generalSettings.setTenantId(TenantId.SYS_TENANT_ID); generalSettings.setKey("general"); ObjectNode node = objectMapper.createObjectNode(); node.put("baseUrl", "http://localhost:8080"); @@ -273,6 +276,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings); AdminSettings mailSettings = new AdminSettings(); + mailSettings.setTenantId(TenantId.SYS_TENANT_ID); mailSettings.setKey("mail"); node = objectMapper.createObjectNode(); node.put("mailFrom", "ThingsBoard "); diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 306102c66c..e428d55d91 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; @@ -121,6 +122,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private ApiUsageStateService apiUsageStateService; + @Lazy @Autowired private QueueService queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index c6576097ee..9bfa86b3ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -117,6 +118,7 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private TenantProfileService tenantProfileService; + @Lazy @Autowired private QueueService queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 2806c35cda..9e60f69620 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -53,7 +53,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -135,6 +135,15 @@ public class DefaultTbClusterService implements TbClusterService { toCoreMsgs.incrementAndGet(); } + @Override + public void pushMsgToVersionControl(TenantId tenantId, TransportProtos.ToVersionControlServiceMsg msg, TbQueueCallback callback) { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId); + log.trace("PUSHING msg: {} to:{}", msg, tpi); + producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback); + //TODO: ashvayka + toCoreMsgs.incrementAndGet(); + } + @Override public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId); @@ -355,13 +364,14 @@ public class DefaultTbClusterService implements TbClusterService { private void broadcast(ComponentLifecycleMsg msg) { byte[] msgBytes = encodingService.encode(msg); TbQueueProducer> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer(); - Set tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE)); + Set tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE); EntityType entityType = msg.getEntityId().getEntityType(); if (entityType.equals(EntityType.TENANT) || entityType.equals(EntityType.TENANT_PROFILE) || entityType.equals(EntityType.DEVICE_PROFILE) || entityType.equals(EntityType.API_USAGE_STATE) || (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED) + || entityType.equals(EntityType.ENTITY_VIEW) || entityType.equals(EntityType.EDGE)) { TbQueueProducer> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer(); Set tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE); @@ -432,12 +442,12 @@ public class DefaultTbClusterService implements TbClusterService { sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); otaPackageStateService.update(device, old); if (!created && notifyEdge) { - sendNotificationMsgToEdgeService(device.getTenantId(), null, device.getId(), null, null, EdgeEventActionType.UPDATED); + sendNotificationMsgToEdge(device.getTenantId(), null, device.getId(), null, null, EdgeEventActionType.UPDATED); } } @Override - public void sendNotificationMsgToEdgeService(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action) { + public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action) { if (!edgesEnabled) { return; } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 2ca47d689f..a3bae8bc4e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -28,6 +28,7 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.MsgType; @@ -36,7 +37,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; @@ -75,6 +76,8 @@ import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbLocalSubscriptionService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; +import org.thingsboard.server.service.sync.vc.GitVersionControlQueueService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import javax.annotation.PostConstruct; @@ -117,6 +120,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService> usageStatsConsumer; private final TbQueueConsumer> firmwareStatesConsumer; @@ -138,7 +142,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray()); if (actorMsg.isPresent()) { @@ -322,6 +332,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService counters = new ArrayList<>(); @@ -70,6 +72,7 @@ public class TbCoreConsumerStats { this.subscriptionMsgCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS)); this.toCoreNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS)); this.edgeNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, EDGE_NOTIFICATIONS)); + this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES)); } private StatsCounter register(StatsCounter counter){ @@ -112,6 +115,11 @@ public class TbCoreConsumerStats { edgeNotificationsCounter.increment(); } + public void log(TransportProtos.DeviceActivityProto msg) { + totalCounter.increment(); + deviceActivitiesCounter.increment(); + } + public void log(TransportProtos.SubscriptionMgrMsgProto msg) { totalCounter.increment(); subscriptionMsgCounter.increment(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 997674753c..98fb95ad15 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -37,7 +37,7 @@ import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; @@ -160,7 +160,7 @@ public abstract class AbstractConsumerService getRuleChainOutputLabels(TenantId tenantId, RuleChainId ruleChainId) { RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId); @@ -167,11 +171,13 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement } @Override - public RuleChain save(RuleChain ruleChain, User user) throws ThingsboardException { + public RuleChain save(RuleChain ruleChain, User user) throws Exception { TenantId tenantId = ruleChain.getTenantId(); ActionType actionType = ruleChain.getId() == null ? ActionType.ADDED : ActionType.UPDATED; - RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); try { + RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain)); + vcService.autoCommit(user, savedRuleChain.getId()); + if (RuleChainType.CORE.equals(savedRuleChain.getType())) { tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedRuleChain.getId(), actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); @@ -220,9 +226,10 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement } @Override - public RuleChain saveDefaultByName(TenantId tenantId, DefaultRuleChainCreateRequest request, User user) throws IOException { + public RuleChain saveDefaultByName(TenantId tenantId, DefaultRuleChainCreateRequest request, User user) throws Exception { try { RuleChain savedRuleChain = installScripts.createDefaultRuleChain(tenantId, request.getName()); + vcService.autoCommit(user, savedRuleChain.getId()); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedRuleChain.getId(), ComponentLifecycleEvent.CREATED); notificationEntityService.logEntityAction(tenantId, savedRuleChain.getId(), savedRuleChain, ActionType.ADDED, user); return savedRuleChain; @@ -266,7 +273,7 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement @Override public RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChain ruleChain, RuleChainMetaData ruleChainMetaData, - boolean updateRelated, User user) throws ThingsboardException { + boolean updateRelated, User user) throws Exception { RuleChainId ruleChainId = ruleChain.getId(); RuleChainId ruleChainMetaDataId = ruleChainMetaData.getRuleChainId(); try { @@ -280,6 +287,15 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement updatedRuleChains = Collections.emptyList(); } + if (updatedRuleChains.isEmpty()) { + vcService.autoCommit(user, ruleChainMetaData.getRuleChainId()); + } else { + List uuids = new ArrayList<>(updatedRuleChains.size() + 1); + uuids.add(ruleChainMetaData.getRuleChainId().getId()); + updatedRuleChains.forEach(rc -> uuids.add(rc.getId().getId())); + vcService.autoCommit(user, EntityType.RULE_CHAIN, uuids); + } + RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaDataId)); if (RuleChainType.CORE.equals(ruleChain.getType())) { diff --git a/application/src/main/java/org/thingsboard/server/service/rule/TbRuleChainService.java b/application/src/main/java/org/thingsboard/server/service/rule/TbRuleChainService.java index 79817667f3..a8c3fed780 100644 --- a/application/src/main/java/org/thingsboard/server/service/rule/TbRuleChainService.java +++ b/application/src/main/java/org/thingsboard/server/service/rule/TbRuleChainService.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.service.entitiy.SimpleTbEntityService; -import java.io.IOException; import java.util.List; import java.util.Set; @@ -39,12 +38,12 @@ public interface TbRuleChainService extends SimpleTbEntityService { List updateRelatedRuleChains(TenantId tenantId, RuleChainId ruleChainId, RuleChainUpdateResult result); - RuleChain saveDefaultByName(TenantId tenantId, DefaultRuleChainCreateRequest request, User user) throws IOException; + RuleChain saveDefaultByName(TenantId tenantId, DefaultRuleChainCreateRequest request, User user) throws Exception; RuleChain setRootRuleChain(TenantId tenantId, RuleChain ruleChain, User user) throws ThingsboardException; RuleChainMetaData saveRuleChainMetaData(TenantId tenantId, RuleChain ruleChain, RuleChainMetaData ruleChainMetaData, - boolean updateRelated, User user) throws ThingsboardException; + boolean updateRelated, User user) throws Exception; RuleChain assignRuleChainToEdge(TenantId tenantId, RuleChain ruleChain, Edge edge, User user) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java index 63d3233928..419a60b3b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java @@ -63,7 +63,7 @@ public class SmsTwoFaProvider extends OtpBasedTwoFaProvider { public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { @Override public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { return deviceSessionsCacheEntry.toByteArray(); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java new file mode 100644 index 0000000000..b2abe2a581 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.apiusage.RateLimitService; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesExportImportService implements EntitiesExportImportService { + + private final Map> exportServices = new HashMap<>(); + private final Map> importServices = new HashMap<>(); + + private final EntityActionService entityActionService; + private final RelationService relationService; + private final RateLimitService rateLimitService; + + protected static final List SUPPORTED_ENTITY_TYPES = List.of( + EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, + EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE, + EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE + ); + + + @Override + public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) { + throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + + EntityType entityType = entityId.getEntityType(); + EntityExportService> exportService = getExportService(entityType); + + return exportService.getExportData(ctx, entityId); + } + + @Override + public , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException { + if (!rateLimitService.checkEntityImportLimit(ctx.getTenantId())) { + throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); + } + if (exportData.getEntity() == null || exportData.getEntity().getId() == null) { + throw new DataValidationException("Invalid entity data"); + } + + EntityType entityType = exportData.getEntityType(); + EntityImportService> importService = getImportService(entityType); + + EntityImportResult importResult = importService.importEntity(ctx, exportData); + ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); + + ctx.addReferenceCallback(importResult.getSaveReferencesCallback()); + ctx.addEventCallback(importResult.getSendEventsCallback()); + return importResult; + } + + @Override + public void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException { + for (ThrowingRunnable saveReferencesCallback : ctx.getReferenceCallbacks()) { + saveReferencesCallback.run(); + } + + relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); + + for (EntityRelation relation : ctx.getRelations()) { + entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, + ActionType.RELATION_ADD_OR_UPDATE, null, relation); + } + } + + + @Override + public Comparator getEntityTypeComparatorForImport() { + return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); + } + + + @SuppressWarnings("unchecked") + private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + EntityExportService exportService = exportServices.get(entityType); + if (exportService == null) { + throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); + } + return (EntityExportService) exportService; + } + + @SuppressWarnings("unchecked") + private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + EntityImportService importService = importServices.get(entityType); + if (importService == null) { + throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); + } + return (EntityImportService) importService; + } + + @Autowired + private void setExportServices(DefaultEntityExportService defaultExportService, + Collection> exportServices) { + exportServices.stream() + .sorted(Comparator.comparing(exportService -> exportService.getSupportedEntityTypes().size(), Comparator.reverseOrder())) + .forEach(exportService -> { + exportService.getSupportedEntityTypes().forEach(entityType -> { + this.exportServices.put(entityType, exportService); + }); + }); + SUPPORTED_ENTITY_TYPES.forEach(entityType -> { + this.exportServices.putIfAbsent(entityType, defaultExportService); + }); + } + + @Autowired + private void setImportServices(Collection> importServices) { + importServices.forEach(entityImportService -> { + this.importServices.put(entityImportService.getEntityType(), entityImportService); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java new file mode 100644 index 0000000000..8657528dac --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Comparator; + +public interface EntitiesExportImportService { + + , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId) throws ThingsboardException; + + , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; + + + void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException; + + Comparator getEntityTypeComparatorForImport(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java new file mode 100644 index 0000000000..6e06144ae2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -0,0 +1,210 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.permission.AccessControlService; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultExportableEntitiesService implements ExportableEntitiesService { + + private final Map> daos = new HashMap<>(); + private final Map> removers = new HashMap<>(); + + private final AccessControlService accessControlService; + + + @Override + public , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) { + EntityType entityType = externalId.getEntityType(); + Dao dao = getDao(entityType); + + E entity = null; + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + entity = exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId()); + } + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + + return entity; + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) { + E entity = findEntityById(id); + + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + return entity; + } + + @Override + public , I extends EntityId> E findEntityById(I id) { + EntityType entityType = id.getEntityType(); + Dao dao = getDao(entityType); + if (dao == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + + return dao.findById(TenantId.SYS_TENANT_ID, id.getId()); + } + + @Override + public , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) { + Dao dao = getDao(entityType); + + E entity = null; + + if (dao instanceof ExportableEntityDao) { + ExportableEntityDao exportableEntityDao = (ExportableEntityDao) dao; + try { + entity = exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name); + } catch (UnsupportedOperationException ignored) { + } + } + if (entity == null || !belongsToTenant(entity, tenantId)) { + return null; + } + + return entity; + } + + @Override + public , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) { + ExportableEntityDao dao = getExportableEntityDao(entityType); + if (dao != null) { + return dao.findByTenantId(tenantId.getId(), pageLink); + } else { + return new PageData<>(); + } + } + + @Override + public I getExternalIdByInternal(I internalId) { + ExportableEntityDao dao = getExportableEntityDao(internalId.getEntityType()); + if (dao != null) { + return dao.getExternalIdByInternal(internalId); + } else { + return null; + } + } + + private boolean belongsToTenant(HasId entity, TenantId tenantId) { + return tenantId.equals(((HasTenantId) entity).getTenantId()); + } + + + @Override + public void removeById(TenantId tenantId, I id) { + EntityType entityType = id.getEntityType(); + BiConsumer entityRemover = removers.get(entityType); + if (entityRemover == null) { + throw new IllegalArgumentException("Unsupported entity type " + entityType); + } + entityRemover.accept(tenantId, id); + } + + private > ExportableEntityDao getExportableEntityDao(EntityType entityType) { + Dao dao = getDao(entityType); + if (dao instanceof ExportableEntityDao) { + return (ExportableEntityDao) dao; + } else { + return null; + } + } + + @SuppressWarnings("unchecked") + private Dao getDao(EntityType entityType) { + return (Dao) daos.get(entityType); + } + + @Autowired + private void setDaos(Collection> daos) { + daos.forEach(dao -> { + if (dao.getEntityType() != null) { + this.daos.put(dao.getEntityType(), dao); + } + }); + } + + @Autowired + private void setRemovers(CustomerService customerService, AssetService assetService, RuleChainService ruleChainService, + DashboardService dashboardService, DeviceProfileService deviceProfileService, + DeviceService deviceService, WidgetsBundleService widgetsBundleService) { + removers.put(EntityType.CUSTOMER, (tenantId, entityId) -> { + customerService.deleteCustomer(tenantId, (CustomerId) entityId); + }); + removers.put(EntityType.ASSET, (tenantId, entityId) -> { + assetService.deleteAsset(tenantId, (AssetId) entityId); + }); + removers.put(EntityType.RULE_CHAIN, (tenantId, entityId) -> { + ruleChainService.deleteRuleChainById(tenantId, (RuleChainId) entityId); + }); + removers.put(EntityType.DASHBOARD, (tenantId, entityId) -> { + dashboardService.deleteDashboard(tenantId, (DashboardId) entityId); + }); + removers.put(EntityType.DEVICE_PROFILE, (tenantId, entityId) -> { + deviceProfileService.deleteDeviceProfile(tenantId, (DeviceProfileId) entityId); + }); + removers.put(EntityType.DEVICE, (tenantId, entityId) -> { + deviceService.deleteDevice(tenantId, (DeviceId) entityId); + }); + removers.put(EntityType.WIDGETS_BUNDLE, (tenantId, entityId) -> { + widgetsBundleService.deleteWidgetsBundle(tenantId, (WidgetsBundleId) entityId); + }); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java new file mode 100644 index 0000000000..505d1b2a3f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting; + +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +public interface EntityExportService, D extends EntityExportData> { + + D getExportData(EntitiesExportCtx ctx, I entityId) throws ThingsboardException; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java new file mode 100644 index 0000000000..0a99979e8c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; + +public interface ExportableEntitiesService { + + , I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId); + + , I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id); + + , I extends EntityId> E findEntityById(I id); + + , I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name); + + , I extends EntityId> PageData findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink); + + I getExternalIdByInternal(I internalId); + + void removeById(TenantId tenantId, I id); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java new file mode 100644 index 0000000000..8875491d42 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class AssetExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, Asset asset, EntityExportData exportData) { + asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.ASSET); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java new file mode 100644 index 0000000000..169b7f273c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { + + @Override + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData) throws ThingsboardException { + setRelatedEntities(ctx, entity, (D) exportData); + super.setAdditionalExportData(ctx, entity, exportData); + } + + protected void setRelatedEntities(EntitiesExportCtx ctx, E mainEntity, D exportData) { + } + + protected D newExportData() { + return (D) new EntityExportData(); + } + + ; + + public abstract Set getSupportedEntityTypes(); + + protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet) { + JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java new file mode 100644 index 0000000000..9a3b195b57 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.common.util.RegexUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Set; +import java.util.UUID; + +@Service +@TbCoreComponent +public class DashboardExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, Dashboard dashboard, EntityExportData exportData) { + if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) { + dashboard.getAssignedCustomers().forEach(customerInfo -> { + customerInfo.setCustomerId(getExternalIdOrElseInternal(ctx, customerInfo.getCustomerId())); + }); + } + for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) { + replaceUuidsRecursively(ctx, entityAlias, Collections.emptySet()); + } + for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { + replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id")); + } + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DASHBOARD); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java new file mode 100644 index 0000000000..7e236370df --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -0,0 +1,184 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@Primary +public class DefaultEntityExportService, D extends EntityExportData> implements EntityExportService { + + @Autowired + @Lazy + protected ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + @Autowired + private AttributesService attributesService; + + @Override + public final D getExportData(EntitiesExportCtx ctx, I entityId) throws ThingsboardException { + D exportData = newExportData(); + + E entity = exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entityId); + if (entity == null) { + throw new IllegalArgumentException(entityId.getEntityType() + " [" + entityId.getId() + "] not found"); + } + + exportData.setEntity(entity); + exportData.setEntityType(entityId.getEntityType()); + setAdditionalExportData(ctx, entity, exportData); + + var externalId = entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + ctx.putExternalId(entityId, externalId); + entity.setId(externalId); + entity.setTenantId(null); + + return exportData; + } + + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData) throws ThingsboardException { + var exportSettings = ctx.getSettings(); + if (exportSettings.isExportRelations()) { + List relations = exportRelations(ctx, entity); + relations.forEach(relation -> { + relation.setFrom(getExternalIdOrElseInternal(ctx, relation.getFrom())); + relation.setTo(getExternalIdOrElseInternal(ctx, relation.getTo())); + }); + exportData.setRelations(relations); + } + if (exportSettings.isExportAttributes()) { + Map> attributes = exportAttributes(ctx, entity); + exportData.setAttributes(attributes); + } + } + + private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { + List relations = new ArrayList<>(); + + List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + relations.addAll(inboundRelations); + + List outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + relations.addAll(outboundRelations); + return relations; + } + + private Map> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { + List scopes; + if (entity.getId().getEntityType() == EntityType.DEVICE) { + scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + } else { + scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + } + Map> attributes = new LinkedHashMap<>(); + scopes.forEach(scope -> { + try { + attributes.put(scope, attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream() + .map(attribute -> { + AttributeExportData attributeExportData = new AttributeExportData(); + attributeExportData.setKey(attribute.getKey()); + attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs()); + attributeExportData.setStrValue(attribute.getStrValue().orElse(null)); + attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null)); + attributeExportData.setLongValue(attribute.getLongValue().orElse(null)); + attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null)); + attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null)); + return attributeExportData; + }) + .collect(Collectors.toList())); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + return attributes; + } + + protected ID getExternalIdOrElseInternal(EntitiesExportCtx ctx, ID internalId) { + if (internalId == null || internalId.isNullUid()) return internalId; + var result = ctx.getExternalId(internalId); + if (result == null) { + result = Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId)) + .orElse(internalId); + ctx.putExternalId(internalId, result); + } + return result; + } + + protected UUID getExternalIdOrElseInternalByUuid(EntitiesExportCtx ctx, UUID internalUuid) { + for (EntityType entityType : EntityType.values()) { + EntityId internalId; + try { + internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid); + } catch (Exception e) { + continue; + } + EntityId externalId = ctx.getExternalId(internalId); + if (externalId != null) { + return externalId.getId(); + } + } + for (EntityType entityType : EntityType.values()) { + EntityId internalId; + try { + internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid); + } catch (Exception e) { + continue; + } + EntityId externalId = exportableEntitiesService.getExternalIdByInternal(internalId); + if (externalId != null) { + ctx.putExternalId(internalId, externalId); + return externalId.getId(); + } + } + return internalUuid; + } + + protected D newExportData() { + return (D) new EntityExportData(); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java new file mode 100644 index 0000000000..bc5136bee6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceExportService extends BaseEntityExportService { + + private final DeviceCredentialsService deviceCredentialsService; + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, Device device, DeviceExportData exportData) { + device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId())); + device.setDeviceProfileId(getExternalIdOrElseInternal(ctx, device.getDeviceProfileId())); + if (ctx.getSettings().isExportCredentials()) { + var credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId()); + credentials.setId(null); + credentials.setDeviceId(null); + exportData.setCredentials(credentials); + } + } + + @Override + protected DeviceExportData newExportData() { + return new DeviceExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java new file mode 100644 index 0000000000..46423c66ef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class DeviceProfileExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData) { + deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId())); + deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.DEVICE_PROFILE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java new file mode 100644 index 0000000000..3fff890d3a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.Set; + +@Service +@TbCoreComponent +public class EntityViewExportService extends BaseEntityExportService> { + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, EntityView entityView, EntityExportData exportData) { + entityView.setEntityId(getExternalIdOrElseInternal(ctx, entityView.getEntityId())); + entityView.setCustomerId(getExternalIdOrElseInternal(ctx, entityView.getCustomerId())); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.ENTITY_VIEW); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java new file mode 100644 index 0000000000..802a03ab94 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.common.util.RegexUtils; + +import java.util.Collections; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainExportService extends BaseEntityExportService { + + private final RuleChainService ruleChainService; + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData) { + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), ruleChain.getId()); + Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()) + .forEach(ruleNode -> { + ruleNode.setRuleChainId(null); + ctx.putExternalId(ruleNode.getId(), ruleNode.getExternalId()); + ruleNode.setId(ctx.getExternalId(ruleNode.getId())); + ruleNode.setCreatedTime(0); + ruleNode.setExternalId(null); + replaceUuidsRecursively(ctx, ruleNode.getConfiguration(), Collections.emptySet()); + }); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(getExternalIdOrElseInternal(ctx, ruleChainConnectionInfo.getTargetRuleChainId())); + }); + exportData.setMetaData(metaData); + if (ruleChain.getFirstRuleNodeId() != null) { + ruleChain.setFirstRuleNodeId(ctx.getExternalId(ruleChain.getFirstRuleNodeId())); + } + } + + @Override + protected RuleChainExportData newExportData() { + return new RuleChainExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.RULE_CHAIN); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java new file mode 100644 index 0000000000..d3dd23910f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.exporting.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; + +import java.util.List; +import java.util.Set; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class WidgetsBundleExportService extends BaseEntityExportService { + + private final WidgetTypeService widgetTypeService; + + @Override + protected void setRelatedEntities(EntitiesExportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData) { + if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) { + throw new IllegalArgumentException("Export of system Widget Bundles is not allowed"); + } + + List widgets = widgetTypeService.findWidgetTypesDetailsByTenantIdAndBundleAlias(ctx.getTenantId(), widgetsBundle.getAlias()); + exportData.setWidgets(widgets); + } + + @Override + protected WidgetsBundleExportData newExportData() { + return new WidgetsBundleExportData(); + } + + @Override + public Set getSupportedEntityTypes() { + return Set.of(EntityType.WIDGETS_BUNDLE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java new file mode 100644 index 0000000000..9437f21e60 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +public interface EntityImportService, D extends EntityExportData> { + + EntityImportResult importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException; + + EntityType getEntityType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 81a0068a4d..b92e100401 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -44,7 +44,6 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.controller.BaseController; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.service.action.EntityActionService; -import org.thingsboard.server.service.importing.BulkImportRequest.ColumnMapping; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.AccessControlService; @@ -67,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -165,7 +163,7 @@ public abstract class AbstractBulkImportService data) { + private void saveKvs(SecurityUser user, E entity, Map data) { Arrays.stream(BulkImportColumnType.values()) .filter(BulkImportColumnType::isKv) .map(kvType -> { @@ -249,7 +247,7 @@ public abstract class AbstractBulkImportService columnsMappings = request.getMapping().getColumns(); + List columnsMappings = request.getMapping().getColumns(); return records.stream() .map(record -> { EntityData entityData = new EntityData(); @@ -280,7 +278,7 @@ public abstract class AbstractBulkImportService fields = new LinkedHashMap<>(); - private final Map kvs = new LinkedHashMap<>(); + private final Map kvs = new LinkedHashMap<>(); private int lineNumber; } diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java index 96075eb4c8..24b566e631 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java index 9f8195ca45..e8eac6a9ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java index 651aedeb0b..0626c8e690 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java index 45c2551be2..d48e9a3d23 100644 --- a/application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.importing; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java new file mode 100644 index 0000000000..ac9a38aa36 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class AssetImportService extends BaseEntityImportService> { + + private final AssetService assetService; + + @Override + protected void setOwner(TenantId tenantId, Asset asset, IdProvider idProvider) { + asset.setTenantId(tenantId); + asset.setCustomerId(idProvider.getInternalId(asset.getCustomerId())); + } + + @Override + protected Asset prepare(EntitiesImportCtx ctx, Asset asset, Asset old, EntityExportData exportData, IdProvider idProvider) { + return asset; + } + + @Override + protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData exportData, IdProvider idProvider) { + return assetService.saveAsset(asset); + } + + @Override + protected void onEntitySaved(User user, Asset savedAsset, Asset oldAsset) throws ThingsboardException { + super.onEntitySaved(user, savedAsset, oldAsset); + if (oldAsset != null) { + entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + protected Asset deepCopy(Asset asset) { + return new Asset(asset); + } + + @Override + protected void cleanupForComparison(Asset e) { + super.cleanupForComparison(e); + if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) { + e.setCustomerId(null); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java new file mode 100644 index 0000000000..d46305a41d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -0,0 +1,396 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.api.client.util.Objects; +import com.google.common.util.concurrent.FutureCallback; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasCustomerId; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { + + @Autowired + @Lazy + private ExportableEntitiesService exportableEntitiesService; + @Autowired + private RelationService relationService; + @Autowired + private TelemetrySubscriptionService tsSubService; + @Autowired + protected EntityActionService entityActionService; + @Autowired + protected TbClusterService clusterService; + @Autowired + protected TbNotificationEntityService entityNotificationService; + + @Transactional(rollbackFor = Exception.class) + @Override + public EntityImportResult importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { + EntityImportResult importResult = new EntityImportResult<>(); + ctx.setCurrentImportResult(importResult); + importResult.setEntityType(getEntityType()); + IdProvider idProvider = new IdProvider(ctx, importResult); + + E entity = exportData.getEntity(); + entity.setExternalId(entity.getId()); + + E existingEntity = findExistingEntity(ctx, entity, idProvider); + importResult.setOldEntity(existingEntity); + + setOwner(ctx.getTenantId(), entity, idProvider); + if (existingEntity == null) { + entity.setId(null); + } else { + entity.setId(existingEntity.getId()); + entity.setCreatedTime(existingEntity.getCreatedTime()); + } + + E prepared = prepare(ctx, entity, existingEntity, exportData, idProvider); + + boolean saveOrUpdate = existingEntity == null || compare(ctx, exportData, prepared, existingEntity); + + if (saveOrUpdate) { + E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider); + boolean created = existingEntity == null; + importResult.setCreated(created); + importResult.setUpdated(!created); + importResult.setSavedEntity(savedEntity); + ctx.putInternalId(exportData.getExternalId(), savedEntity.getId()); + } else { + importResult.setSavedEntity(existingEntity); + ctx.putInternalId(exportData.getExternalId(), existingEntity.getId()); + importResult.setUpdatedRelatedEntities(updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider)); + } + + processAfterSaved(ctx, importResult, exportData, idProvider); + + return importResult; + } + + protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) { + return false; + } + + @Override + public abstract EntityType getEntityType(); + + protected abstract void setOwner(TenantId tenantId, E entity, IdProvider idProvider); + + protected abstract E prepare(EntitiesImportCtx ctx, E entity, E oldEntity, D exportData, IdProvider idProvider); + + protected boolean compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) { + var newCopy = deepCopy(prepared); + var existingCopy = deepCopy(existing); + cleanupForComparison(newCopy); + cleanupForComparison(existingCopy); + var result = !newCopy.equals(existingCopy); + if (result) { + log.debug("[{}] Found update.", prepared.getId()); + log.debug("[{}] From: {}", prepared.getId(), newCopy); + log.debug("[{}] To: {}", prepared.getId(), existingCopy); + } + return result; + } + + protected abstract E deepCopy(E e); + + protected void cleanupForComparison(E e) { + e.setTenantId(null); + e.setCreatedTime(0); + } + + protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider); + + + protected void processAfterSaved(EntitiesImportCtx ctx, EntityImportResult importResult, D exportData, IdProvider idProvider) throws ThingsboardException { + E savedEntity = importResult.getSavedEntity(); + E oldEntity = importResult.getOldEntity(); + + if (importResult.isCreated() || importResult.isUpdated()) { + importResult.addSendEventsCallback(() -> onEntitySaved(ctx.getUser(), savedEntity, oldEntity)); + } + + if (ctx.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(ctx, exportData.getRelations(), importResult, idProvider); + } + if (ctx.isSaveAttributes() && exportData.getAttributes() != null) { + if (exportData.getAttributes().values().stream().anyMatch(d -> !d.isEmpty())) { + importResult.setUpdatedRelatedEntities(true); + } + importAttributes(ctx.getUser(), exportData.getAttributes(), importResult); + } + } + + private void importRelations(EntitiesImportCtx ctx, List relations, EntityImportResult importResult, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + for (EntityRelation relation : relations) { + if (!relation.getTo().equals(entity.getId())) { + relation.setTo(idProvider.getInternalId(relation.getTo())); + } + if (!relation.getFrom().equals(entity.getId())) { + relation.setFrom(idProvider.getInternalId(relation.getFrom())); + } + } + + Map relationsMap = new LinkedHashMap<>(); + relations.forEach(r -> relationsMap.put(r, r)); + + if (importResult.getOldEntity() != null) { + List existingRelations = new ArrayList<>(); + existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + + for (EntityRelation existingRelation : existingRelations) { + EntityRelation relation = relationsMap.get(existingRelation); + if (relation == null) { + importResult.setUpdatedRelatedEntities(true); + relationService.deleteRelation(tenantId, existingRelation); + importResult.addSendEventsCallback(() -> { + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null, + ActionType.RELATION_DELETED, null, existingRelation); + }); + } else if (Objects.equal(relation.getAdditionalInfo(), existingRelation.getAdditionalInfo())) { + relationsMap.remove(relation); + } + } + } + if (!relationsMap.isEmpty()) { + importResult.setUpdatedRelatedEntities(true); + ctx.addRelations(relationsMap.values()); + } + }); + } + + private void importAttributes(User user, Map> attributes, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + attributes.forEach((scope, attributesExportData) -> { + List attributeKvEntries = attributesExportData.stream() + .map(attributeExportData -> { + KvEntry kvEntry; + String key = attributeExportData.getKey(); + if (attributeExportData.getStrValue() != null) { + kvEntry = new StringDataEntry(key, attributeExportData.getStrValue()); + } else if (attributeExportData.getBooleanValue() != null) { + kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue()); + } else if (attributeExportData.getDoubleValue() != null) { + kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue()); + } else if (attributeExportData.getLongValue() != null) { + kvEntry = new LongDataEntry(key, attributeExportData.getLongValue()); + } else if (attributeExportData.getJsonValue() != null) { + kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue()); + } else { + throw new IllegalArgumentException("Invalid attribute export data"); + } + return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs()); + }) + .collect(Collectors.toList()); + // fixme: attributes are saved outside the transaction + tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void unused) { + } + + @Override + public void onFailure(Throwable thr) { + log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr); + } + }); + }); + }); + } + + protected void onEntitySaved(User user, E savedEntity, E oldEntity) throws ThingsboardException { + entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, + savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), + oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null); + } + + + @SuppressWarnings("unchecked") + protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) { + return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId())) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId()))) + .or(() -> { + if (ctx.isFindExistingByName()) { + return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName())); + } else { + return Optional.empty(); + } + }) + .orElse(null); + } + + @SuppressWarnings("unchecked") + private HasId findInternalEntity(TenantId tenantId, ID externalId) { + return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) + .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) + .orElseThrow(() -> new MissingEntityException(externalId)); + } + + + @SuppressWarnings("unchecked") + @RequiredArgsConstructor + protected class IdProvider { + private final EntitiesImportCtx ctx; + private final EntityImportResult importResult; + + public ID getInternalId(ID externalId) { + return getInternalId(externalId, true); + } + + public ID getInternalId(ID externalId, boolean throwExceptionIfNotFound) { + if (externalId == null || externalId.isNullUid()) return null; + + EntityId localId = ctx.getInternalId(externalId); + if (localId != null) { + return (ID) localId; + } + + HasId entity; + try { + entity = findInternalEntity(ctx.getTenantId(), externalId); + } catch (Exception e) { + if (throwExceptionIfNotFound) { + throw e; + } else { + importResult.setUpdatedAllExternalIds(false); + return null; + } + } + ctx.putInternalId(externalId, entity.getId()); + return entity.getId(); + } + + public Optional getInternalIdByUuid(UUID externalUuid, boolean fetchAllUUIDs, Set hints) { + if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty(); + + for (EntityType entityType : EntityType.values()) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (!externalIdOpt.isPresent()) { + continue; + } + EntityId internalId = ctx.getInternalId(externalIdOpt.get()); + if (internalId != null) { + return Optional.of(internalId); + } + } + + if (fetchAllUUIDs) { + for (EntityType entityType : hints) { + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; + } + for (EntityType entityType : EntityType.values()) { + if (hints.contains(entityType)) { + continue; + } + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; + } + } + + importResult.setUpdatedAllExternalIds(false); + return Optional.empty(); + } + + private Optional lookupInDb(UUID externalUuid, EntityType entityType) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (externalIdOpt.isEmpty() || ctx.isNotFound(externalIdOpt.get())) { + return Optional.empty(); + } + EntityId internalId = getInternalId(externalIdOpt.get(), false); + if (internalId != null) { + return Optional.of(internalId); + } else { + ctx.registerNotFound(externalIdOpt.get()); + } + return Optional.empty(); + } + + private Optional buildEntityId(EntityType entityType, UUID externalUuid) { + try { + return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, externalUuid)); + } catch (Exception e) { + return Optional.empty(); + } + } + + } + + protected T getOldEntityField(O oldEntity, Function getter) { + return oldEntity == null ? null : getter.apply(oldEntity); + } + + protected void replaceIdsRecursively(EntitiesImportCtx ctx, IdProvider idProvider, JsonNode entityAlias, Set skipFieldsSet, LinkedHashSet hints) { + JacksonUtil.replaceUuidsRecursively(entityAlias, skipFieldsSet, + uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints).map(EntityId::getId).orElse(uuid)); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java new file mode 100644 index 0000000000..be1ef9ed8c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.customer.CustomerDao; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class CustomerImportService extends BaseEntityImportService> { + + private final CustomerService customerService; + private final CustomerDao customerDao; + + @Override + protected void setOwner(TenantId tenantId, Customer customer, IdProvider idProvider) { + customer.setTenantId(tenantId); + } + + @Override + protected Customer prepare(EntitiesImportCtx ctx, Customer customer, Customer old, EntityExportData exportData, IdProvider idProvider) { + if (customer.isPublic()) { + Customer publicCustomer = customerService.findOrCreatePublicCustomer(ctx.getTenantId()); + publicCustomer.setExternalId(customer.getExternalId()); + return publicCustomer; + } else { + return customer; + } + } + + @Override + protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData exportData, IdProvider idProvider) { + if (!customer.isPublic()) { + return customerService.saveCustomer(customer); + } else { + return customerDao.save(ctx.getTenantId(), customer); + } + } + + @Override + protected Customer deepCopy(Customer customer) { + return new Customer(customer); + } + + @Override + protected void onEntitySaved(User user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException { + super.onEntitySaved(user, savedCustomer, oldCustomer); + if (oldCustomer != null) { + entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java new file mode 100644 index 0000000000..9e6083c61c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import com.fasterxml.jackson.databind.JsonNode; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DashboardImportService extends BaseEntityImportService> { + + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.DASHBOARD, EntityType.DEVICE, EntityType.ASSET)); + + private final DashboardService dashboardService; + + + @Override + protected void setOwner(TenantId tenantId, Dashboard dashboard, IdProvider idProvider) { + dashboard.setTenantId(tenantId); + } + + @Override + protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard, IdProvider idProvider) { + Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard, idProvider); + if (existingDashboard == null && ctx.isFindExistingByName()) { + existingDashboard = dashboardService.findTenantDashboardsByTitle(ctx.getTenantId(), dashboard.getName()).stream().findFirst().orElse(null); + } + return existingDashboard; + } + + @Override + protected Dashboard prepare(EntitiesImportCtx ctx, Dashboard dashboard, Dashboard old, EntityExportData exportData, IdProvider idProvider) { + for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) { + replaceIdsRecursively(ctx, idProvider, entityAlias, Collections.emptySet(), HINTS); + } + for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) { + replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id"), HINTS); + } + return dashboard; + } + + @Override + protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData exportData, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); + + Set assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream() + .peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId()))) + .collect(Collectors.toSet()); + + if (dashboard.getId() == null) { + dashboard.setAssignedCustomers(assignedCustomers); + dashboard = dashboardService.saveDashboard(dashboard); + for (ShortCustomerInfo customerInfo : assignedCustomers) { + dashboard = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerInfo.getCustomerId()); + } + } else { + Set existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers()) + .orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + Set newAssignedCustomers = assignedCustomers.stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet()); + + Set toUnassign = new HashSet<>(existingAssignedCustomers); + toUnassign.removeAll(newAssignedCustomers); + for (CustomerId customerId : toUnassign) { + assignedCustomers = dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); + } + + Set toAssign = new HashSet<>(newAssignedCustomers); + toAssign.removeAll(existingAssignedCustomers); + for (CustomerId customerId : toAssign) { + assignedCustomers = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers(); + } + dashboard.setAssignedCustomers(assignedCustomers); + dashboard = dashboardService.saveDashboard(dashboard); + } + return dashboard; + } + + @Override + protected Dashboard deepCopy(Dashboard dashboard) { + return new Dashboard(dashboard); + } + + @Override + protected void onEntitySaved(User user, Dashboard savedDashboard, Dashboard oldDashboard) throws ThingsboardException { + super.onEntitySaved(user, savedDashboard, oldDashboard); + if (oldDashboard != null) { + entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java new file mode 100644 index 0000000000..cf46bcee24 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceImportService extends BaseEntityImportService { + + private final DeviceService deviceService; + private final DeviceCredentialsService credentialsService; + + @Override + protected void setOwner(TenantId tenantId, Device device, IdProvider idProvider) { + device.setTenantId(tenantId); + device.setCustomerId(idProvider.getInternalId(device.getCustomerId())); + } + + @Override + protected Device prepare(EntitiesImportCtx ctx, Device device, Device old, DeviceExportData exportData, IdProvider idProvider) { + device.setDeviceProfileId(idProvider.getInternalId(device.getDeviceProfileId())); + device.setFirmwareId(getOldEntityField(old, Device::getFirmwareId)); + device.setSoftwareId(getOldEntityField(old, Device::getSoftwareId)); + return device; + } + + @Override + protected Device deepCopy(Device d) { + return new Device(d); + } + + @Override + protected void cleanupForComparison(Device e) { + super.cleanupForComparison(e); + if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) { + e.setCustomerId(null); + } + } + + @Override + protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) { + if (exportData.getCredentials() != null && ctx.isSaveCredentials()) { + exportData.getCredentials().setId(null); + exportData.getCredentials().setDeviceId(null); + return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials()); + } else { + return deviceService.saveDevice(device); + } + } + + @Override + protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, Device prepared, DeviceExportData exportData, IdProvider idProvider) { + boolean updated = super.updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider); + var credentials = exportData.getCredentials(); + if (credentials != null && ctx.isSaveCredentials()) { + var existing = credentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), prepared.getId()); + credentials.setId(existing.getId()); + credentials.setDeviceId(prepared.getId()); + if (!existing.equals(credentials)) { + credentialsService.updateDeviceCredentials(ctx.getTenantId(), credentials); + updated = true; + } + } + return updated; + } + + @Override + protected void onEntitySaved(User user, Device savedDevice, Device oldDevice) throws ThingsboardException { + super.onEntitySaved(user, savedDevice, oldDevice); + clusterService.onDeviceUpdated(savedDevice, oldDevice); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java new file mode 100644 index 0000000000..890c3cfc07 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -0,0 +1,93 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ota.OtaPackageStateService; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Objects; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class DeviceProfileImportService extends BaseEntityImportService> { + + private final DeviceProfileService deviceProfileService; + private final OtaPackageStateService otaPackageStateService; + + @Override + protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) { + deviceProfile.setTenantId(tenantId); + } + + @Override + protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData exportData, IdProvider idProvider) { + deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId())); + deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId())); + deviceProfile.setFirmwareId(getOldEntityField(old, DeviceProfile::getFirmwareId)); + deviceProfile.setSoftwareId(getOldEntityField(old, DeviceProfile::getSoftwareId)); + return deviceProfile; + } + + @Override + protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData exportData, IdProvider idProvider) { + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + @Override + protected void onEntitySaved(User user, DeviceProfile savedDeviceProfile, DeviceProfile oldDeviceProfile) throws ThingsboardException { + super.onEntitySaved(user, savedDeviceProfile, oldDeviceProfile); + clusterService.onDeviceProfileChange(savedDeviceProfile, null); + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedDeviceProfile.getId(), + oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + otaPackageStateService.update(savedDeviceProfile, + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()), + oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId())); + } + + @Override + protected DeviceProfile deepCopy(DeviceProfile deviceProfile) { + return new DeviceProfile(deviceProfile); + } + + @Override + protected void cleanupForComparison(DeviceProfile deviceProfile) { + super.cleanupForComparison(deviceProfile); + deviceProfile.setFirmwareId(null); + deviceProfile.setSoftwareId(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java new file mode 100644 index 0000000000..06e0e56237 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class EntityViewImportService extends BaseEntityImportService> { + + private final EntityViewService entityViewService; + + @Lazy + @Autowired + private TbEntityViewService tbEntityViewService; + + @Override + protected void setOwner(TenantId tenantId, EntityView entityView, IdProvider idProvider) { + entityView.setTenantId(tenantId); + entityView.setCustomerId(idProvider.getInternalId(entityView.getCustomerId())); + } + + @Override + protected EntityView prepare(EntitiesImportCtx ctx, EntityView entityView, EntityView old, EntityExportData exportData, IdProvider idProvider) { + entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId())); + return entityView; + } + + @Override + protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData exportData, IdProvider idProvider) { + return entityViewService.saveEntityView(entityView); + } + + @Override + protected void onEntitySaved(User user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException { + tbEntityViewService.updateEntityViewAttributes(user.getTenantId(), savedEntityView, oldEntityView, user); + super.onEntitySaved(user, savedEntityView, oldEntityView); + if (oldEntityView != null) { + entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedEntityView.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + protected EntityView deepCopy(EntityView entityView) { + return new EntityView(entityView); + } + + @Override + protected void cleanupForComparison(EntityView e) { + super.cleanupForComparison(e); + if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) { + e.setCustomerId(null); + } + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java new file mode 100644 index 0000000000..1e869429d8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +public class ImportServiceException extends RuntimeException{ + private static final long serialVersionUID = -4932715239522125041L; +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java new file mode 100644 index 0000000000..a0a961bfef --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; + +public class MissingEntityException extends ImportServiceException { + + private static final long serialVersionUID = 3669135386955906022L; + @Getter + private final EntityId entityId; + + public MissingEntityException(EntityId entityId) { + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java new file mode 100644 index 0000000000..500aa27d3a --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -0,0 +1,148 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.rule.RuleNodeDao; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Slf4j +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class RuleChainImportService extends BaseEntityImportService { + + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); + + private final RuleChainService ruleChainService; + private final RuleNodeDao ruleNodeDao; + + @Override + protected void setOwner(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) { + ruleChain.setTenantId(tenantId); + } + + @Override + protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain, IdProvider idProvider) { + RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain, idProvider); + if (existingRuleChain == null && ctx.isFindExistingByName()) { + existingRuleChain = ruleChainService.findTenantRuleChainsByTypeAndName(ctx.getTenantId(), ruleChain.getType(), ruleChain.getName()).stream().findFirst().orElse(null); + } + return existingRuleChain; + } + + @Override + protected RuleChain prepare(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChain old, RuleChainExportData exportData, IdProvider idProvider) { + RuleChainMetaData metaData = exportData.getMetaData(); + List ruleNodes = Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList()); + if (old != null) { + List nodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + List existing = ruleNodeDao.findByExternalIds(old.getId(), nodeIds); + existing.forEach(node -> ctx.putInternalId(node.getExternalId(), node.getId())); + ruleNodes.forEach(node -> { + node.setRuleChainId(old.getId()); + node.setExternalId(node.getId()); + node.setId((RuleNodeId) ctx.getInternalId(node.getId())); + }); + } else { + ruleNodes.forEach(node -> { + node.setRuleChainId(null); + node.setExternalId(node.getId()); + node.setId(null); + }); + } + + ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), Collections.emptySet(), HINTS)); + Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList()) + .forEach(ruleChainConnectionInfo -> { + ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false)); + }); + if (ruleChain.getFirstRuleNodeId() != null) { + ruleChain.setFirstRuleNodeId((RuleNodeId) ctx.getInternalId(ruleChain.getFirstRuleNodeId())); + } + return ruleChain; + } + + @Override + protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider) { + ruleChain = ruleChainService.saveRuleChain(ruleChain); + if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) { + exportData.getMetaData().setRuleChainId(ruleChain.getId()); + ruleChainService.saveRuleChainMetaData(ctx.getTenantId(), exportData.getMetaData()); + return ruleChainService.findRuleChainById(ctx.getTenantId(), ruleChain.getId()); + } else { + return ruleChain; + } + } + + @Override + protected boolean compare(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) { + boolean different = super.compare(ctx, exportData, prepared, existing); + if (!different) { + RuleChainMetaData newMD = exportData.getMetaData(); + RuleChainMetaData existingMD = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), prepared.getId()); + existingMD.setRuleChainId(null); + different = newMD.equals(existingMD); + } + return different; + } + + @Override + protected void onEntitySaved(User user, RuleChain savedRuleChain, RuleChain oldRuleChain) throws ThingsboardException { + super.onEntitySaved(user, savedRuleChain, oldRuleChain); + if (savedRuleChain.getType() == RuleChainType.CORE) { + clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(), + oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + } else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) { + entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED); + } + } + + @Override + protected RuleChain deepCopy(RuleChain ruleChain) { + return new RuleChain(ruleChain); + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java new file mode 100644 index 0000000000..d0758a14d2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie.importing.impl; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.WidgetsBundleId; +import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData; +import org.thingsboard.server.common.data.widget.BaseWidgetType; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetTypeInfo; +import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.dao.widget.WidgetTypeService; +import org.thingsboard.server.dao.widget.WidgetsBundleService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; + +import java.util.Map; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +public class WidgetsBundleImportService extends BaseEntityImportService { + + private final WidgetsBundleService widgetsBundleService; + private final WidgetTypeService widgetTypeService; + + @Override + protected void setOwner(TenantId tenantId, WidgetsBundle widgetsBundle, IdProvider idProvider) { + widgetsBundle.setTenantId(tenantId); + } + + @Override + protected WidgetsBundle prepare(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundle old, WidgetsBundleExportData exportData, IdProvider idProvider) { + return widgetsBundle; + } + + @Override + protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider) { + WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle); + if (widgetsBundle.getId() == null) { + for (WidgetTypeDetails widget : exportData.getWidgets()) { + widget.setId(null); + widget.setTenantId(ctx.getTenantId()); + widget.setBundleAlias(savedWidgetsBundle.getAlias()); + widgetTypeService.saveWidgetType(widget); + } + } else { + Map existingWidgets = widgetTypeService.findWidgetTypesInfosByTenantIdAndBundleAlias(ctx.getTenantId(), savedWidgetsBundle.getAlias()).stream() + .collect(Collectors.toMap(BaseWidgetType::getAlias, w -> w)); + for (WidgetTypeDetails widget : exportData.getWidgets()) { + WidgetTypeInfo existingWidget; + if ((existingWidget = existingWidgets.remove(widget.getAlias())) != null) { + widget.setId(existingWidget.getId()); + widget.setCreatedTime(existingWidget.getCreatedTime()); + } else { + widget.setId(null); + } + widget.setTenantId(ctx.getTenantId()); + widget.setBundleAlias(savedWidgetsBundle.getAlias()); + widgetTypeService.saveWidgetType(widget); + } + existingWidgets.values().stream() + .map(BaseWidgetType::getId) + .forEach(widgetTypeId -> widgetTypeService.deleteWidgetType(ctx.getTenantId(), widgetTypeId)); + } + return savedWidgetsBundle; + } + + @Override + protected void onEntitySaved(User user, WidgetsBundle savedWidgetsBundle, WidgetsBundle oldWidgetsBundle) throws ThingsboardException { + super.onEntitySaved(user, savedWidgetsBundle, oldWidgetsBundle); + entityNotificationService.notifySendMsgToEdgeService(user.getTenantId(), savedWidgetsBundle.getId(), + oldWidgetsBundle == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED); + } + + @Override + protected WidgetsBundle deepCopy(WidgetsBundle widgetsBundle) { + return new WidgetsBundle(widgetsBundle); + } + + @Override + public EntityType getEntityType() { + return EntityType.WIDGETS_BUNDLE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..4fe9db4ba1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -0,0 +1,621 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.springframework.transaction.support.TransactionTemplate; +import org.thingsboard.common.util.DonAsynchron; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.TbStopWatch; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityLoadError; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; +import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx; +import org.thingsboard.server.service.sync.vc.data.ReimportTask; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.google.common.util.concurrent.Futures.transform; +import static com.google.common.util.concurrent.Futures.transformAsync; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + + private final TbRepositorySettingsService repositorySettingsService; + private final TbAutoCommitSettingsService autoCommitSettingsService; + private final GitVersionControlQueueService gitServiceQueue; + private final EntitiesExportImportService exportImportService; + private final ExportableEntitiesService exportableEntitiesService; + private final TbNotificationEntityService entityNotificationService; + private final TransactionTemplate transactionTemplate; + private final TbTransactionalCache taskCache; + + private ListeningExecutorService executor; + + @Value("${vc.thread_pool_size:4}") + private int threadPoolSize; + + @PostConstruct + public void init() { + executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class)); + } + + @PreDestroy + public void shutdown() { + if (executor != null) { + executor.shutdownNow(); + } + } + + @SuppressWarnings("UnstableApiUsage") + @Override + public ListenableFuture saveEntitiesVersion(User user, VersionCreateRequest request) throws Exception { + var pendingCommit = gitServiceQueue.prepareCommit(user, request); + DonAsynchron.withCallback(pendingCommit, commit -> { + cachePut(commit.getTxId(), new VersionCreationResult()); + try { + List> gitFutures = new ArrayList<>(); + switch (request.getType()) { + case SINGLE_ENTITY: { + handleSingleEntityRequest(new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request)); + break; + } + case COMPLEX: { + handleComplexRequest(new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request)); + break; + } + } + var resultFuture = Futures.transformAsync(Futures.allAsList(gitFutures), f -> gitServiceQueue.push(commit), executor); + DonAsynchron.withCallback(resultFuture, result -> cachePut(commit.getTxId(), result), e -> processCommitError(user, request, commit, e), executor); + } catch (Exception e) { + processCommitError(user, request, commit, e); + } + }, t -> log.debug("[{}] Failed to prepare the commit: {}", user.getId(), request, t)); + + return transform(pendingCommit, CommitGitRequest::getTxId, MoreExecutors.directExecutor()); + } + + @Override + public VersionCreationResult getVersionCreateStatus(User user, UUID requestId) throws ThingsboardException { + return getStatus(user, requestId, VersionControlTaskCacheEntry::getExportResult); + } + + @Override + public VersionLoadResult getVersionLoadStatus(User user, UUID requestId) throws ThingsboardException { + return getStatus(user, requestId, VersionControlTaskCacheEntry::getImportResult); + } + + private T getStatus(User user, UUID requestId, Function getter) throws ThingsboardException { + var cacheEntry = taskCache.get(requestId); + if (cacheEntry == null || cacheEntry.get() == null) { + throw new ThingsboardException(ThingsboardErrorCode.ITEM_NOT_FOUND); + } else { + var entry = cacheEntry.get(); + var result = getter.apply(entry); + if (result == null) { + throw new ThingsboardException(ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } else { + return result; + } + } + } + + private void handleSingleEntityRequest(SimpleEntitiesExportCtx ctx) throws Exception { + ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId())); + } + + private void handleComplexRequest(ComplexEntitiesExportCtx parentCtx) { + ComplexVersionCreateRequest request = parentCtx.getRequest(); + request.getEntityTypes().forEach((entityType, config) -> { + EntityTypeExportCtx ctx = new EntityTypeExportCtx(parentCtx, config, request.getSyncStrategy(), entityType); + if (ctx.isOverwrite()) { + ctx.add(gitServiceQueue.deleteAll(ctx.getCommit(), entityType)); + } + + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink) + , 100, entity -> { + try { + ctx.add(saveEntityData(ctx, entity.getId())); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId))); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + }); + } + + private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId); + return gitServiceQueue.addToCommit(ctx.getCommit(), entityData); + } + + @Override + public ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, externalId, pageLink); + } + + @Override + public ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, entityType, pageLink); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception { + return gitServiceQueue.listVersions(tenantId, branch, pageLink); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId, entityType); + } + + @Override + public ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId); + } + + @SuppressWarnings({"UnstableApiUsage", "rawtypes"}) + @Override + public UUID loadEntitiesVersion(User user, VersionLoadRequest request) throws Exception { + EntitiesImportCtx ctx = new EntitiesImportCtx(UUID.randomUUID(), user, request.getVersionId()); + cachePut(ctx.getRequestId(), VersionLoadResult.empty()); + switch (request.getType()) { + case SINGLE_ENTITY: { + SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + VersionLoadConfig config = versionLoadRequest.getConfig(); + ListenableFuture future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + DonAsynchron.withCallback(future, + entityData -> doInTemplate(ctx, request, c -> loadSingleEntity(c, config, entityData)), + e -> processLoadError(ctx, e), executor); + break; + } + case ENTITY_TYPE: { + EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; + executor.submit(() -> doInTemplate(ctx, request, c -> loadMultipleEntities(c, versionLoadRequest))); + break; + } + default: + throw new IllegalArgumentException("Unsupported version load request"); + } + + return ctx.getRequestId(); + } + + private VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function function) { + try { + VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx)); + for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { + throwingRunnable.run(); + } + result.setDone(true); + return cachePut(ctx.getRequestId(), result); + } catch (LoadEntityException e) { + return cachePut(ctx.getRequestId(), onError(e.getData(), e.getCause())); + } catch (Exception e) { + log.info("[{}] Failed to process request [{}] due to: ", ctx.getTenantId(), request, e); + return cachePut(ctx.getRequestId(), VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage()))); + } + } + + private VersionLoadResult loadSingleEntity(EntitiesImportCtx ctx, VersionLoadConfig config, EntityExportData entityData) { + try { + ctx.setSettings(EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) + .findExistingByName(false) + .build()); + ctx.setFinalImportAttempt(true); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); + + exportImportService.saveReferencesAndRelations(ctx); + + return VersionLoadResult.success(EntityTypeLoadResult.builder() + .entityType(importResult.getEntityType()) + .created(importResult.getOldEntity() == null ? 1 : 0) + .updated(importResult.getOldEntity() != null ? 1 : 0) + .deleted(0) + .build()); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + } + + @SneakyThrows + private VersionLoadResult loadMultipleEntities(EntitiesImportCtx ctx, EntityTypeVersionLoadRequest request) { + var sw = TbStopWatch.create("before"); + + List entityTypes = request.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); + for (EntityType entityType : entityTypes) { + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); + sw.startNew("Entities " + entityType.name()); + ctx.setSettings(getEntityImportSettings(request, entityType)); + importEntities(ctx, entityType); + persistToCache(ctx); + } + + sw.startNew("Reimport"); + reimport(ctx); + persistToCache(ctx); + + sw.startNew("Remove Others"); + request.getEntityTypes().keySet().stream() + .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> removeOtherEntities(ctx, entityType)); + persistToCache(ctx); + + sw.startNew("References and Relations"); + exportImportService.saveReferencesAndRelations(ctx); + + sw.stop(); + for (var task : sw.getTaskInfo()) { + log.info("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); + } + log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); + return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values())); + } + + private EntityImportSettings getEntityImportSettings(EntityTypeVersionLoadRequest request, EntityType entityType) { + var config = request.getEntityTypes().get(entityType); + return EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) + .saveCredentials(config.isLoadCredentials()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void importEntities(EntitiesImportCtx ctx, EntityType entityType) { + int limit = 100; + int offset = 0; + List entityDataList; + do { + try { + entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + for (EntityExportData entityData : entityDataList) { + EntityExportData reimportBackup = JacksonUtil.clone(entityData); + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); + EntityImportResult importResult; + try { + importResult = exportImportService.importEntity(ctx, entityData); + } catch (Exception e) { + throw new LoadEntityException(entityData, e); + } + if (!importResult.isUpdatedAllExternalIds()) { + ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(reimportBackup, ctx.getSettings())); + continue; + } + + registerResult(ctx, entityType, importResult); + ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) + .add(importResult.getSavedEntity().getId()); + } + offset += limit; + } while (entityDataList.size() == limit); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void reimport(EntitiesImportCtx ctx) { + ctx.setFinalImportAttempt(true); + ctx.getToReimport().forEach((externalId, task) -> { + try { + EntityExportData entityData = task.getData(); + var settings = task.getSettings(); + ctx.setSettings(settings); + EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); + + registerResult(ctx, externalId.getEntityType(), importResult); + ctx.getImportedEntities().computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>()) + .add(importResult.getSavedEntity().getId()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) { + exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId()); + + ctx.addEventCallback(() -> { + entityNotificationService.notifyDeleteEntity(ctx.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, ctx.getUser()); + }); + ctx.registerDeleted(entityType); + } + }); + } + + private VersionLoadResult onError(EntityExportData entityData, Throwable e) { + return analyze(e, entityData).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage()))); + } + + private Optional analyze(Throwable e, EntityExportData entityData) { + if (e == null) { + return Optional.empty(); + } else { + if (e instanceof DeviceCredentialsValidationException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId()))); + } else if (e instanceof MissingEntityException) { + return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId()))); + } else { + return analyze(e.getCause(), entityData); + } + } + } + + @Override + public ListenableFuture compareEntityDataToVersion(User user, String branch, EntityId entityId, String versionId) throws Exception { + HasId entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId); + if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type"); + + EntityId externalId = ((ExportableEntity) entity).getExternalId(); + if (externalId == null) externalId = entityId; + + return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + otherVersion -> { + SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder() + .exportRelations(otherVersion.hasRelations()) + .exportAttributes(otherVersion.hasAttributes()) + .exportCredentials(otherVersion.hasCredentials()) + .build()); + EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId); + return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), + rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + }, MoreExecutors.directExecutor()); + } + + @Override + public ListenableFuture getEntityDataInfo(User user, EntityId entityId, String versionId) { + return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId), + entity -> new EntityDataInfo(entity.hasRelations(), entity.hasAttributes(), entity.hasCredentials()), MoreExecutors.directExecutor()); + } + + + @Override + public ListenableFuture> listBranches(TenantId tenantId) throws Exception { + return gitServiceQueue.listBranches(tenantId); + } + + @Override + public RepositorySettings getVersionControlSettings(TenantId tenantId) { + return repositorySettingsService.get(tenantId); + } + + @Override + public ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings) { + var restoredSettings = this.repositorySettingsService.restore(tenantId, versionControlSettings); + try { + var future = gitServiceQueue.initRepository(tenantId, restoredSettings); + return Futures.transform(future, f -> repositorySettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor()); + } catch (Exception e) { + log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e); + throw new RuntimeException("Failed to init repository!", e); + } + } + + @Override + public ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception { + if (repositorySettingsService.delete(tenantId)) { + return gitServiceQueue.clearRepository(tenantId); + } else { + return Futures.immediateFuture(null); + } + } + + @Override + public ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws ThingsboardException { + settings = this.repositorySettingsService.restore(tenantId, settings); + try { + return gitServiceQueue.testRepository(tenantId, settings); + } catch (Exception e) { + throw new ThingsboardException(String.format("Unable to access repository: %s", getCauseMessage(e)), + ThingsboardErrorCode.GENERAL); + } + } + + @Override + public ListenableFuture autoCommit(User user, EntityId entityId) throws Exception { + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } + var entityType = entityId.getEntityType(); + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest(); + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; + } + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setEntityId(entityId); + vcr.setConfig(autoCommitConfig); + return saveEntitiesVersion(user, vcr); + } + + @Override + public ListenableFuture autoCommit(User user, EntityType entityType, List entityIds) throws Exception { + var repositorySettings = repositorySettingsService.get(user.getTenantId()); + if (repositorySettings == null) { + return Futures.immediateFuture(null); + } + var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId()); + if (autoCommitSettings == null) { + return Futures.immediateFuture(null); + } + AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType); + if (autoCommitConfig == null) { + return Futures.immediateFuture(null); + } + var autoCommitBranchName = autoCommitConfig.getBranch(); + if (StringUtils.isEmpty(autoCommitBranchName)) { + autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits"; + } + ComplexVersionCreateRequest vcr = new ComplexVersionCreateRequest(); + vcr.setBranch(autoCommitBranchName); + vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000)); + vcr.setSyncStrategy(SyncStrategy.MERGE); + + EntityTypeVersionCreateConfig vcrConfig = new EntityTypeVersionCreateConfig(); + vcrConfig.setEntityIds(entityIds); + vcr.setEntityTypes(Collections.singletonMap(entityType, vcrConfig)); + return saveEntitiesVersion(user, vcr); + } + + private String getCauseMessage(Exception e) { + String message; + if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) { + message = e.getCause().getMessage(); + } else { + message = e.getMessage(); + } + return message; + } + + private void registerResult(EntitiesImportCtx ctx, EntityType entityType, EntityImportResult importResult) { + if (importResult.isCreated()) { + ctx.registerResult(entityType, true); + } else if (importResult.isUpdated() || importResult.isUpdatedRelatedEntities()) { + ctx.registerResult(entityType, false); + } + } + + private void processCommitError(User user, VersionCreateRequest request, CommitGitRequest commit, Throwable e) { + log.debug("[{}] Failed to prepare the commit: {}", user.getId(), request, e); + cachePut(commit.getTxId(), new VersionCreationResult(e.getMessage())); + } + + private void processLoadError(EntitiesImportCtx ctx, Throwable e) { + log.debug("[{}] Failed to load the commit: {}", ctx.getRequestId(), ctx.getVersionId(), e); + cachePut(ctx.getRequestId(), VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage()))); + } + + private void cachePut(UUID requestId, VersionCreationResult result) { + taskCache.put(requestId, VersionControlTaskCacheEntry.newForExport(result)); + } + + private VersionLoadResult cachePut(UUID requestId, VersionLoadResult result) { + log.debug("[{}] Cache put: {}", requestId, result); + taskCache.put(requestId, VersionControlTaskCacheEntry.newForImport(result)); + return result; + } + + private void persistToCache(EntitiesImportCtx ctx) { + cachePut(ctx.getRequestId(), VersionLoadResult.success(new ArrayList<>(ctx.getResults().values()))); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java new file mode 100644 index 0000000000..628605824e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -0,0 +1,532 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import com.google.protobuf.ByteString; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GenericRepositoryRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.scheduler.SchedulerComponent; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ContentsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.EntityContentGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListBranchesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListEntitiesGitRequest; +import org.thingsboard.server.service.sync.vc.data.ListVersionsGitRequest; +import org.thingsboard.server.service.sync.vc.data.PendingGitRequest; +import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; +import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService { + + private final TbServiceInfoProvider serviceInfoProvider; + private final TbClusterService clusterService; + private final DataDecodingEncodingService encodingService; + private final DefaultEntitiesVersionControlService entitiesVersionControlService; + private final SchedulerComponent scheduler; + + private final Map> pendingRequestMap = new HashMap<>(); + + @Value("${queue.vc.request-timeout:60000}") + private int requestTimeout; + + public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, + DataDecodingEncodingService encodingService, + @Lazy DefaultEntitiesVersionControlService entitiesVersionControlService, + SchedulerComponent scheduler) { + this.serviceInfoProvider = serviceInfoProvider; + this.clusterService = clusterService; + this.encodingService = encodingService; + this.entitiesVersionControlService = entitiesVersionControlService; + this.scheduler = scheduler; + } + + @Override + public ListenableFuture prepareCommit(User user, VersionCreateRequest request) { + SettableFuture future = SettableFuture.create(); + + CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request); + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build() + ).build(), wrap(future, commit)); + return future; + } + + @Override + public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); + String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); + + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJson(entityDataJson).build() + ).build() + ).build(), wrap(future, null)); + return future; + } + + @Override + public ListenableFuture deleteAll(CommitGitRequest commit, EntityType entityType) { + SettableFuture future = SettableFuture.create(); + + String path = getRelativePath(entityType, null); + + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setDeleteMsg( + TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build() + ).build() + ).build(), wrap(future, null)); + + return future; + } + + @Override + public ListenableFuture push(CommitGitRequest commit) { + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setPushMsg( + TransportProtos.PushMsg.newBuilder().build() + ).build() + ).build(), wrap(commit.getFuture())); + + return commit.getFuture(); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink) { + + return listVersions(tenantId, + applyPageLinkParameters( + ListVersionsRequestMsg.newBuilder() + .setBranchName(branch), + pageLink + ).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) { + return listVersions(tenantId, + applyPageLinkParameters( + ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityType.name()), + pageLink + ).build()); + } + + @Override + public ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) { + return listVersions(tenantId, + applyPageLinkParameters( + ListVersionsRequestMsg.newBuilder() + .setBranchName(branch) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()), + pageLink + ).build()); + } + + private ListVersionsRequestMsg.Builder applyPageLinkParameters(ListVersionsRequestMsg.Builder builder, PageLink pageLink) { + builder.setPageSize(pageLink.getPageSize()) + .setPage(pageLink.getPage()); + if (pageLink.getTextSearch() != null) { + builder.setTextSearch(pageLink.getTextSearch()); + } + if (pageLink.getSortOrder() != null) { + if (pageLink.getSortOrder().getProperty() != null) { + builder.setSortProperty(pageLink.getSortOrder().getProperty()); + } + if (pageLink.getSortOrder().getDirection() != null) { + builder.setSortDirection(pageLink.getSortOrder().getDirection().name()); + } + } + return builder; + } + + private ListenableFuture> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) { + ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListVersionRequest(requestMsg)); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .setEntityType(entityType.name()) + .build()); + } + + @Override + public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() + .setBranchName(branch) + .setVersionId(versionId) + .build()); + } + + private ListenableFuture> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) { + ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListEntitiesRequest(requestMsg)); + } + + @Override + public ListenableFuture> listBranches(TenantId tenantId) { + ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); + return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); + } + + @Override + public ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2) { + String path = entityType != null ? getRelativePath(entityType, externalId) : ""; + VersionsDiffGitRequest request = new VersionsDiffGitRequest(tenantId, path, versionId1, versionId2); + return sendRequest(request, builder -> builder.setVersionsDiffRequest(TransportProtos.VersionsDiffRequestMsg.newBuilder() + .setPath(request.getPath()) + .setVersionId1(request.getVersionId1()) + .setVersionId2(request.getVersionId2()) + .build())); + } + + @Override + public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { + ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); + return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() + .setContent1(content1) + .setContent2(content2))); + } + + @Override + @SuppressWarnings("rawtypes") + public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { + EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build() + , wrap(request.getFuture())); + return request.getFuture(); + } + + private void registerAndSend(PendingGitRequest request, + Function enrichFunction, TbQueueCallback callback) { + registerAndSend(request, enrichFunction, null, callback); + } + + private void registerAndSend(PendingGitRequest request, + Function enrichFunction, RepositorySettings settings, TbQueueCallback callback) { + if (!request.getFuture().isDone()) { + pendingRequestMap.putIfAbsent(request.getRequestId(), request); + var requestBody = enrichFunction.apply(newRequestProto(request, settings)); + log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); + clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); + request.setTimeoutTask(scheduler.schedule(() -> { + processTimeout(request.getRequestId()); + }, requestTimeout, TimeUnit.MILLISECONDS)); + } else { + throw new RuntimeException("Future is already done!"); + } + } + + private ListenableFuture sendRequest(PendingGitRequest request, Consumer enrichFunction) { + registerAndSend(request, builder -> { + enrichFunction.accept(builder); + return builder.build(); + }, wrap(request.getFuture())); + return request.getFuture(); + } + + @Override + @SuppressWarnings("rawtypes") + public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { + EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); + + registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() + .setVersionId(versionId) + .setEntityType(entityType.name()) + .setOffset(offset) + .setLimit(limit) + ).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , settings, wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings) { + VoidGitRequest request = new VoidGitRequest(tenantId); + + registerAndSend(request, builder -> builder + .setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , settings, wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public ListenableFuture clearRepository(TenantId tenantId) { + ClearRepositoryGitRequest request = new ClearRepositoryGitRequest(tenantId); + + registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build() + , wrap(request.getFuture())); + + return request.getFuture(); + } + + @Override + public void processResponse(VersionControlResponseMsg vcResponseMsg) { + UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); + PendingGitRequest request = pendingRequestMap.remove(requestId); + if (request == null) { + log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); + return; + } else { + log.debug("[{}] processing response: {}", requestId, vcResponseMsg); + request.getTimeoutTask().cancel(true); + } + var future = request.getFuture(); + if (!StringUtils.isEmpty(vcResponseMsg.getError())) { + future.setException(new RuntimeException(vcResponseMsg.getError())); + } else { + if (vcResponseMsg.hasGenericResponse()) { + future.set(null); + } else if (vcResponseMsg.hasCommitResponse()) { + var commitResponse = vcResponseMsg.getCommitResponse(); + var commitResult = new VersionCreationResult(); + if (commitResponse.getTs() > 0) { + commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor())); + } + commitResult.setAdded(commitResponse.getAdded()); + commitResult.setRemoved(commitResponse.getRemoved()); + commitResult.setModified(commitResponse.getModified()); + commitResult.setDone(true); + ((CommitGitRequest) request).getFuture().set(commitResult); + } else if (vcResponseMsg.hasListBranchesResponse()) { + var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); + ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList().stream().map(this::getBranchInfo).collect(Collectors.toList())); + } else if (vcResponseMsg.hasListEntitiesResponse()) { + var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); + ((ListEntitiesGitRequest) request).getFuture().set( + listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList())); + } else if (vcResponseMsg.hasListVersionsResponse()) { + var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); + ((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse)); + } else if (vcResponseMsg.hasEntityContentResponse()) { + var data = vcResponseMsg.getEntityContentResponse().getData(); + ((EntityContentGitRequest) request).getFuture().set(toData(data)); + } else if (vcResponseMsg.hasEntitiesContentResponse()) { + var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); + ((EntitiesContentGitRequest) request).getFuture() + .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + } else if (vcResponseMsg.hasVersionsDiffResponse()) { + TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); + List entityVersionsDiffList = diffResponse.getDiffList().stream() + .map(diff -> EntityVersionsDiff.builder() + .externalId(EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(diff.getEntityType()), + new UUID(diff.getEntityIdMSB(), diff.getEntityIdLSB()))) + .entityDataAtVersion1(StringUtils.isNotEmpty(diff.getEntityDataAtVersion1()) ? + toData(diff.getEntityDataAtVersion1()) : null) + .entityDataAtVersion2(StringUtils.isNotEmpty(diff.getEntityDataAtVersion2()) ? + toData(diff.getEntityDataAtVersion2()) : null) + .rawDiff(diff.getRawDiff()) + .build()) + .collect(Collectors.toList()); + ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); + } else if (vcResponseMsg.hasContentsDiffResponse()) { + String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); + ((ContentsDiffGitRequest) request).getFuture().set(diff); + } + } + } + + private void processTimeout(UUID requestId) { + PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + if (pendingRequest != null) { + log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout); + pendingRequest.getFuture().setException(new TimeoutException("Request timed out")); + } + } + + private PageData toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) { + var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()); + return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext()); + } + + private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) { + return new EntityVersion(proto.getTs(), proto.getId(), proto.getName(), proto.getAuthor()); + } + + private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) { + return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); + } + + private BranchInfo getBranchInfo(TransportProtos.BranchInfoProto proto) { + return new BranchInfo(proto.getName(), proto.getIsDefault()); + } + + @SuppressWarnings("rawtypes") + @SneakyThrows + private EntityExportData toData(String data) { + return JacksonUtil.fromString(data, EntityExportData.class); + } + + private static TbQueueCallback wrap(SettableFuture future) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static TbQueueCallback wrap(SettableFuture future, T value) { + return new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + future.set(value); + } + + @Override + public void onFailure(Throwable t) { + future.setException(t); + } + }; + } + + private static String getRelativePath(EntityType entityType, EntityId entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private static PrepareMsg getCommitPrepareMsg(User user, VersionCreateRequest request) { + return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName()) + .setBranchName(request.getBranch()).setAuthorName(getAuthorName(user)).setAuthorEmail(user.getEmail()).build(); + } + + private static String getAuthorName(User user) { + List parts = new ArrayList<>(); + if (StringUtils.isNotBlank(user.getFirstName())) { + parts.add(user.getFirstName()); + } + if (StringUtils.isNotBlank(user.getLastName())) { + parts.add(user.getLastName()); + } + if (parts.isEmpty()) { + parts.add(user.getName()); + } + return String.join(" ", parts); + } + + private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest request, RepositorySettings settings) { + var tenantId = request.getTenantId(); + var requestId = request.getRequestId(); + var builder = ToVersionControlServiceMsg.newBuilder() + .setNodeId(serviceInfoProvider.getServiceId()) + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setRequestIdMSB(requestId.getMostSignificantBits()) + .setRequestIdLSB(requestId.getLeastSignificantBits()); + RepositorySettings vcSettings = settings; + if (vcSettings == null && request.requiresSettings()) { + vcSettings = entitiesVersionControlService.getVersionControlSettings(tenantId); + } + if (vcSettings != null) { + builder.setVcSettings(ByteString.copyFrom(encodingService.encode(vcSettings))); + } else if (request.requiresSettings()) { + throw new RuntimeException("No entity version control settings provisioned!"); + } + return builder; + } + + private CommitRequestMsg.Builder buildCommitRequest(CommitGitRequest commit) { + return CommitRequestMsg.newBuilder().setTxId(commit.getTxId().toString()); + } +} + diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java new file mode 100644 index 0000000000..8bc2049e33 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; +import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; + +import java.util.List; +import java.util.UUID; + +public interface EntitiesVersionControlService { + + ListenableFuture saveEntitiesVersion(User user, VersionCreateRequest request) throws Exception; + + VersionCreationResult getVersionCreateStatus(User user, UUID requestId) throws ThingsboardException; + + ListenableFuture> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception; + + ListenableFuture> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception; + + ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception; + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; + + ListenableFuture> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + UUID loadEntitiesVersion(User user, VersionLoadRequest request) throws Exception; + + VersionLoadResult getVersionLoadStatus(User user, UUID requestId) throws ThingsboardException; + + ListenableFuture compareEntityDataToVersion(User user, String branch, EntityId entityId, String versionId) throws Exception; + + ListenableFuture> listBranches(TenantId tenantId) throws Exception; + + RepositorySettings getVersionControlSettings(TenantId tenantId); + + ListenableFuture saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings); + + ListenableFuture deleteVersionControlSettings(TenantId tenantId) throws Exception; + + ListenableFuture checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception; + + ListenableFuture autoCommit(User user, EntityId entityId) throws Exception; + + ListenableFuture autoCommit(User user, EntityType entityType, List entityIds) throws Exception; + + ListenableFuture getEntityDataInfo(User user, EntityId entityId, String versionId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java new file mode 100644 index 0000000000..a1e64da8f1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; + +import java.util.List; + +public interface GitVersionControlQueueService { + + ListenableFuture prepareCommit(User user, VersionCreateRequest request); + + ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData); + + ListenableFuture deleteAll(CommitGitRequest pendingCommit, EntityType entityType); + + ListenableFuture push(CommitGitRequest commit); + + ListenableFuture> listVersions(TenantId tenantId, String branch, PageLink pageLink); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink); + + ListenableFuture> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + ListenableFuture> listBranches(TenantId tenantId); + + ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId); + + ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit); + + ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); + + ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); + + ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); + + ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); + + ListenableFuture clearRepository(TenantId tenantId); + + void processResponse(VersionControlResponseMsg vcResponseMsg); +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java new file mode 100644 index 0000000000..a1b036d37b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@SuppressWarnings("rawtypes") +public class LoadEntityException extends RuntimeException { + + private static final long serialVersionUID = -1749719992370409504L; + @Getter + private final EntityExportData data; + + public LoadEntityException(EntityExportData data, Throwable cause) { + super(cause); + this.data = data; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java new file mode 100644 index 0000000000..3a674eece0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import java.io.Serializable; + +public abstract class TbAbstractVersionControlSettingsService { + + private final String settingsKey; + private final AdminSettingsService adminSettingsService; + private final TbTransactionalCache cache; + private final Class clazz; + + public TbAbstractVersionControlSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache, Class clazz, String settingsKey) { + this.adminSettingsService = adminSettingsService; + this.cache = cache; + this.clazz = clazz; + this.settingsKey = settingsKey; + } + + public T get(TenantId tenantId) { + return cache.getAndPutInTransaction(tenantId, () -> { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, settingsKey); + if (adminSettings != null) { + try { + return JacksonUtil.convertValue(adminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load " + settingsKey + " settings!", e); + } + } + return null; + }, true); + } + + public T save(TenantId tenantId, T settings) { + AdminSettings adminSettings = adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, settingsKey); + if (adminSettings == null) { + adminSettings = new AdminSettings(); + adminSettings.setKey(settingsKey); + adminSettings.setTenantId(tenantId); + } + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings); + T savedSettings; + try { + savedSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), clazz); + } catch (Exception e) { + throw new RuntimeException("Failed to load auto commit settings!", e); + } + //API calls to adminSettingsService are not in transaction, so we can simply evict the cache. + cache.evict(tenantId); + return savedSettings; + } + + public boolean delete(TenantId tenantId) { + boolean result = adminSettingsService.deleteAdminSettingsByTenantIdAndKey(tenantId, settingsKey); + cache.evict(tenantId); + return result; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskCacheEntry.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskCacheEntry.java new file mode 100644 index 0000000000..dcf3ef3a70 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskCacheEntry.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; + +import java.io.Serializable; + +@Data +@AllArgsConstructor +public class VersionControlTaskCacheEntry implements Serializable { + + private static final long serialVersionUID = -7875992200801588119L; + + private VersionCreationResult exportResult; + private VersionLoadResult importResult; + + public static VersionControlTaskCacheEntry newForExport(VersionCreationResult result) { + return new VersionControlTaskCacheEntry(result, null); + } + + public static VersionControlTaskCacheEntry newForImport(VersionLoadResult result) { + return new VersionControlTaskCacheEntry(null, result); + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskCaffeineCache.java new file mode 100644 index 0000000000..e58ceba8c5 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskCaffeineCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("VersionControlTaskCache") +public class VersionControlTaskCaffeineCache extends CaffeineTbTransactionalCache { + + public VersionControlTaskCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.VERSION_CONTROL_TASK_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java new file mode 100644 index 0000000000..be053f6308 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.gen.transport.TransportProtos; + +import java.util.UUID; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("VersionControlTaskCache") +public class VersionControlTaskRedisCache extends RedisTbTransactionalCache { + + public VersionControlTaskRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.VERSION_CONTROL_TASK_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java new file mode 100644 index 0000000000..47b9e5f5d2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public AutoCommitSettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.AUTO_COMMIT_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java new file mode 100644 index 0000000000..f88b1cf6bd --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("AutoCommitSettingsCache") +public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache { + + public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java new file mode 100644 index 0000000000..b6e8d45ec2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbAutoCommitSettingsService extends TbAbstractVersionControlSettingsService implements TbAutoCommitSettingsService { + + public static final String SETTINGS_KEY = "autoCommitSettings"; + + public DefaultTbAutoCommitSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, AutoCommitSettings.class, SETTINGS_KEY); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java new file mode 100644 index 0000000000..51978481e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.autocommit; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbAutoCommitSettingsService { + + AutoCommitSettings get(TenantId tenantId); + + AutoCommitSettings save(TenantId tenantId, AutoCommitSettings settings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java new file mode 100644 index 0000000000..09245eb895 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; + +public class ClearRepositoryGitRequest extends VoidGitRequest { + + public ClearRepositoryGitRequest(TenantId tenantId) { + super(tenantId); + } + + public boolean requiresSettings() { + return false; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java new file mode 100644 index 0000000000..a989439d10 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +public class CommitGitRequest extends PendingGitRequest { + + @Getter + private final UUID txId; + private final VersionCreateRequest request; + + public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) { + super(tenantId); + this.txId = UUID.randomUUID(); + this.request = request; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java new file mode 100644 index 0000000000..e8577b049d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; + +import java.util.HashMap; +import java.util.Map; + +public class ComplexEntitiesExportCtx extends EntitiesExportCtx { + + private final Map settings = new HashMap<>(); + + public ComplexEntitiesExportCtx(User user, CommitGitRequest commit, ComplexVersionCreateRequest request) { + super(user, commit, request); + request.getEntityTypes().forEach((type, config) -> settings.put(type, buildExportSettings(config))); + } + + public EntityExportSettings getSettings(EntityType entityType) { + return settings.get(entityType); + } + + @Override + public EntityExportSettings getSettings() { + throw new RuntimeException("Not implemented. Use EntityTypeExportCtx instead!"); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java new file mode 100644 index 0000000000..7c3fb7a600 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ContentsDiffGitRequest.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; + +@Getter +public class ContentsDiffGitRequest extends PendingGitRequest { + + private final String content1; + private final String content2; + + public ContentsDiffGitRequest(TenantId tenantId, String content1, String content2) { + super(tenantId); + this.content1 = content1; + this.content2 = content2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java new file mode 100644 index 0000000000..ad653edd0c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesContentGitRequest.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +import java.util.List; + +@Getter +public class EntitiesContentGitRequest extends PendingGitRequest> { + + private final String versionId; + private final EntityType entityType; + + public EntitiesContentGitRequest(TenantId tenantId, String versionId, EntityType entityType) { + super(tenantId); + this.versionId = versionId; + this.entityType = entityType; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java new file mode 100644 index 0000000000..4c89c91271 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -0,0 +1,88 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Data +public abstract class EntitiesExportCtx { + + protected final User user; + protected final CommitGitRequest commit; + protected final R request; + private final List> futures; + private final Map externalIdMap; + + public EntitiesExportCtx(User user, CommitGitRequest commit, R request) { + this.user = user; + this.commit = commit; + this.request = request; + this.futures = new ArrayList<>(); + this.externalIdMap = new HashMap<>(); + } + + protected EntitiesExportCtx(EntitiesExportCtx other) { + this.user = other.getUser(); + this.commit = other.getCommit(); + this.request = other.getRequest(); + this.futures = other.getFutures(); + this.externalIdMap = other.getExternalIdMap(); + } + + public void add(ListenableFuture future) { + futures.add(future); + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + protected static EntityExportSettings buildExportSettings(VersionCreateConfig config) { + return EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + } + + public abstract EntityExportSettings getSettings(); + + @SuppressWarnings("unchecked") + public ID getExternalId(ID internalId) { + var result = externalIdMap.get(internalId); + log.debug("[{}][{}] Local cache {} for id", internalId.getEntityType(), internalId.getId(), result != null ? "hit" : "miss"); + return (ID) result; + } + + public void putExternalId(EntityId internalId, EntityId externalId) { + log.debug("[{}][{}] Local cache put: {}", internalId.getEntityType(), internalId.getId(), externalId); + externalIdMap.put(internalId, externalId != null ? externalId : internalId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java new file mode 100644 index 0000000000..d7b85a1560 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -0,0 +1,143 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +@Slf4j +@Data +public class EntitiesImportCtx { + + private final UUID requestId; + private final User user; + private final String versionId; + + private final Map results = new HashMap<>(); + private final Map> importedEntities = new HashMap<>(); + private final Map toReimport = new HashMap<>(); + private final List referenceCallbacks = new ArrayList<>(); + private final List eventCallbacks = new ArrayList<>(); + private final Map externalToInternalIdMap = new HashMap<>(); + private final Set notFoundIds = new HashSet<>(); + + private final Set relations = new LinkedHashSet<>(); + + private boolean finalImportAttempt = false; + private EntityImportSettings settings; + private EntityImportResult currentImportResult; + + public EntitiesImportCtx(UUID requestId, User user, String versionId) { + this(requestId, user, versionId, null); + } + + public EntitiesImportCtx(UUID requestId, User user, String versionId, EntityImportSettings settings) { + this.requestId = requestId; + this.user = user; + this.versionId = versionId; + this.settings = settings; + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + public boolean isFindExistingByName() { + return getSettings().isFindExistingByName(); + } + + public boolean isUpdateRelations() { + return getSettings().isUpdateRelations(); + } + + public boolean isSaveAttributes() { + return getSettings().isSaveAttributes(); + } + + public boolean isSaveCredentials() { + return getSettings().isSaveCredentials(); + } + + public EntityId getInternalId(EntityId externalId) { + var result = externalToInternalIdMap.get(externalId); + log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); + return result; + } + + public void putInternalId(EntityId externalId, EntityId internalId) { + log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId); + externalToInternalIdMap.put(externalId, internalId); + } + + public void registerResult(EntityType entityType, boolean created) { + EntityTypeLoadResult result = results.computeIfAbsent(entityType, EntityTypeLoadResult::new); + if (created) { + result.setCreated(result.getCreated() + 1); + } else { + result.setUpdated(result.getUpdated() + 1); + } + } + + public void registerDeleted(EntityType entityType) { + EntityTypeLoadResult result = results.computeIfAbsent(entityType, EntityTypeLoadResult::new); + result.setDeleted(result.getDeleted() + 1); + } + + public void addRelations(Collection values) { + relations.addAll(values); + } + + public void addReferenceCallback(ThrowingRunnable tr) { + if (tr != null) { + referenceCallbacks.add(tr); + } + } + + public void addEventCallback(ThrowingRunnable tr) { + if (tr != null) { + eventCallbacks.add(tr); + } + } + + public void registerNotFound(EntityId externalId) { + notFoundIds.add(externalId); + } + + public boolean isNotFound(EntityId externalId) { + return notFoundIds.contains(externalId); + } + + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java new file mode 100644 index 0000000000..6a1d60a065 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityContentGitRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Getter +public class EntityContentGitRequest extends PendingGitRequest { + + private final String versionId; + private final EntityId entityId; + + public EntityContentGitRequest(TenantId tenantId, String versionId, EntityId entityId) { + super(tenantId); + this.versionId = versionId; + this.entityId = entityId; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java new file mode 100644 index 0000000000..80d90590a1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityTypeExportCtx.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.apache.commons.lang3.ObjectUtils; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +public class EntityTypeExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityType entityType; + @Getter + private final boolean overwrite; + @Getter + private final EntityExportSettings settings; + + public EntityTypeExportCtx(EntitiesExportCtx parent, EntityTypeVersionCreateConfig config, SyncStrategy defaultSyncStrategy, EntityType entityType) { + super(parent); + this.entityType = entityType; + this.settings = EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + this.overwrite = ObjectUtils.defaultIfNull(config.getSyncStrategy(), defaultSyncStrategy) == SyncStrategy.OVERWRITE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java new file mode 100644 index 0000000000..4d89efa14d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; + +import java.util.List; + +public class ListBranchesGitRequest extends PendingGitRequest> { + + public ListBranchesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java new file mode 100644 index 0000000000..3fc531735e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListEntitiesGitRequest.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.util.List; + +public class ListEntitiesGitRequest extends PendingGitRequest> { + + public ListEntitiesGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java new file mode 100644 index 0000000000..d40a589646 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListVersionsGitRequest.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; + +public class ListVersionsGitRequest extends PendingGitRequest> { + + public ListVersionsGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java new file mode 100644 index 0000000000..b34806a6ca --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/PendingGitRequest.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import com.google.common.util.concurrent.SettableFuture; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; +import java.util.concurrent.ScheduledFuture; + +@Getter +public class PendingGitRequest { + + private final long createdTime; + private final UUID requestId; + private final TenantId tenantId; + private final SettableFuture future; + @Setter + private ScheduledFuture timeoutTask; + + public PendingGitRequest(TenantId tenantId) { + this.createdTime = System.currentTimeMillis(); + this.requestId = UUID.randomUUID(); + this.tenantId = tenantId; + this.future = SettableFuture.create(); + } + + public boolean requiresSettings() { + return true; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java new file mode 100644 index 0000000000..97432adbb8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; + +@Data +public class ReimportTask { + + private final EntityExportData data; + private final EntityImportSettings settings; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java new file mode 100644 index 0000000000..4cf5d93765 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; + +public class SimpleEntitiesExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityExportSettings settings; + + public SimpleEntitiesExportCtx(User user, CommitGitRequest commit, SingleEntityVersionCreateRequest request) { + this(user, commit, request, request != null ? buildExportSettings(request.getConfig()) : null); + } + + public SimpleEntitiesExportCtx(User user, CommitGitRequest commit, SingleEntityVersionCreateRequest request, EntityExportSettings settings) { + super(user, commit, request); + this.settings = settings; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java new file mode 100644 index 0000000000..e73706609f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionsDiffGitRequest.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; + +import java.util.List; + +@Getter +public class VersionsDiffGitRequest extends PendingGitRequest> { + + private final String path; + private final String versionId1; + private final String versionId2; + + public VersionsDiffGitRequest(TenantId tenantId, String path, String versionId1, String versionId2) { + super(tenantId); + this.path = path; + this.versionId1 = versionId1; + this.versionId2 = versionId2; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java new file mode 100644 index 0000000000..c48bfa7a03 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VoidGitRequest.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.id.TenantId; + +public class VoidGitRequest extends PendingGitRequest { + + public VoidGitRequest(TenantId tenantId) { + super(tenantId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java new file mode 100644 index 0000000000..555490bd06 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/DefaultTbRepositorySettingsService.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService; + +@Service +@TbCoreComponent +public class DefaultTbRepositorySettingsService extends TbAbstractVersionControlSettingsService implements TbRepositorySettingsService { + + public static final String SETTINGS_KEY = "entitiesVersionControl"; + + public DefaultTbRepositorySettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache cache) { + super(adminSettingsService, cache, RepositorySettings.class, SETTINGS_KEY); + } + + @Override + public RepositorySettings restore(TenantId tenantId, RepositorySettings settings) { + RepositorySettings storedSettings = get(tenantId); + if (storedSettings != null) { + RepositoryAuthMethod authMethod = settings.getAuthMethod(); + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(authMethod) && settings.getPassword() == null) { + settings.setPassword(storedSettings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(authMethod) && settings.getPrivateKey() == null) { + settings.setPrivateKey(storedSettings.getPrivateKey()); + if (settings.getPrivateKeyPassword() == null) { + settings.setPrivateKeyPassword(storedSettings.getPrivateKeyPassword()); + } + } + } + return settings; + } + + @Override + public RepositorySettings get(TenantId tenantId) { + RepositorySettings settings = super.get(tenantId); + if (settings != null) { + settings = new RepositorySettings(settings); + } + return settings; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java new file mode 100644 index 0000000000..60b7f50e12 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsCaffeineCache.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("RepositorySettingsCache") +public class RepositorySettingsCaffeineCache extends CaffeineTbTransactionalCache { + + public RepositorySettingsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.REPOSITORY_SETTINGS_CACHE); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java new file mode 100644 index 0000000000..3cccb6d24e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("RepositorySettingsCache") +public class RepositorySettingsRedisCache extends RedisTbTransactionalCache { + + public RepositorySettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java new file mode 100644 index 0000000000..946d06c87c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/TbRepositorySettingsService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.repository; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; + +public interface TbRepositorySettingsService { + + RepositorySettings restore(TenantId tenantId, RepositorySettings versionControlSettings); + + RepositorySettings get(TenantId tenantId); + + RepositorySettings save(TenantId tenantId, RepositorySettings versionControlSettings); + + boolean delete(TenantId tenantId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index 38275c09b6..5571c4badb 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.cluster.TbClusterService; @@ -43,12 +44,12 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; +import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; @@ -75,7 +76,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer private final AttributesService attrService; private final TimeseriesService tsService; - private final EntityViewService entityViewService; + private final TbEntityViewService tbEntityViewService; private final TbApiUsageClient apiUsageClient; private final TbApiUsageStateService apiUsageStateService; @@ -83,7 +84,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer public DefaultTelemetrySubscriptionService(AttributesService attrService, TimeseriesService tsService, - EntityViewService entityViewService, + @Lazy TbEntityViewService tbEntityViewService, TbClusterService clusterService, PartitionService partitionService, TbApiUsageClient apiUsageClient, @@ -91,7 +92,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer super(clusterService, partitionService); this.attrService = attrService; this.tsService = tsService; - this.entityViewService = entityViewService; + this.tbEntityViewService = tbEntityViewService; this.apiUsageClient = apiUsageClient; this.apiUsageStateService = apiUsageStateService; } @@ -182,11 +183,11 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer addMainCallback(saveFuture, callback); addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts)); if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) { - Futures.addCallback(this.entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), + Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId), new FutureCallback>() { @Override public void onSuccess(@Nullable List result) { - if (result != null) { + if (result != null && !result.isEmpty()) { Map> tsMap = new HashMap<>(); for (TsKvEntry entry : ts) { tsMap.computeIfAbsent(entry.getKey(), s -> new ArrayList<>()).add(entry); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 6c4adfc5ae..f8b42ecc1e 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -65,7 +65,7 @@ import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceProvisionService; import org.thingsboard.server.dao.device.DeviceService; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java index 4e3abd8e68..e178055706 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java @@ -32,7 +32,7 @@ import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.action.EntityActionService; @@ -50,7 +50,7 @@ public class AlarmsCleanUpService { @Value("${sql.ttl.alarms.removal_batch_size}") private Integer removalBatchSize; - private final TenantDao tenantDao; + private final TenantService tenantService; private final AlarmDao alarmDao; private final AlarmService alarmService; private final RelationService relationService; @@ -64,7 +64,7 @@ public class AlarmsCleanUpService { PageLink removalBatchRequest = new PageLink(removalBatchSize, 0 ); PageData tenantsIds; do { - tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest); + tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); for (TenantId tenantId : tenantsIds.getData()) { if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { continue; diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index 877987d6f0..1c21cc6029 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -27,7 +27,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -43,7 +43,7 @@ public class RpcCleanUpService { @Value("${sql.ttl.rpc.enabled}") private boolean ttlTaskExecutionEnabled; - private final TenantDao tenantDao; + private final TenantService tenantService; private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; private final RpcDao rpcDao; @@ -54,7 +54,7 @@ public class RpcCleanUpService { PageLink tenantsBatchRequest = new PageLink(10_000, 0); PageData tenantsIds; do { - tenantsIds = tenantDao.findTenantsIds(tenantsBatchRequest); + tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); for (TenantId tenantId : tenantsIds.getData()) { if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { continue; diff --git a/application/src/main/java/org/thingsboard/server/springfox/SpringfoxHandlerProviderBeanPostProcessor.java b/application/src/main/java/org/thingsboard/server/springfox/SpringfoxHandlerProviderBeanPostProcessor.java new file mode 100644 index 0000000000..af03c72004 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/springfox/SpringfoxHandlerProviderBeanPostProcessor.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.springfox; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.stereotype.Component; +import org.springframework.util.ReflectionUtils; +import org.springframework.web.servlet.mvc.method.RequestMappingInfoHandlerMapping; +import org.thingsboard.server.queue.util.TbCoreComponent; +import springfox.documentation.spring.web.plugins.WebMvcRequestHandlerProvider; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.stream.Collectors; + +@Component +//TODO: remove after fixing issue https://github.com/springfox/springfox/issues/3462 or after migration from springfox to springdoc +public class SpringfoxHandlerProviderBeanPostProcessor implements BeanPostProcessor { + + @Override + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + if (bean instanceof WebMvcRequestHandlerProvider) { + customizeSpringfoxHandlerMappings(getHandlerMappings(bean)); + } + return bean; + } + + private void customizeSpringfoxHandlerMappings(List mappings) { + List copy = mappings.stream() + .filter(mapping -> mapping.getPatternParser() == null) + .collect(Collectors.toList()); + mappings.clear(); + mappings.addAll(copy); + } + + @SuppressWarnings("unchecked") + private List getHandlerMappings(Object bean) { + try { + Field field = ReflectionUtils.findField(bean.getClass(), "handlerMappings"); + field.setAccessible(true); + return (List) field.get(bean); + } catch (IllegalArgumentException | IllegalAccessException e) { + throw new IllegalStateException(e); + } + } +} diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index c68d484cd5..4784ad2d43 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,23 +25,31 @@ - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9bb0e8a9ff..f6312a7659 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -394,6 +394,9 @@ cache: tenantProfiles: timeToLiveInMinutes: "${CACHE_SPECS_TENANT_PROFILES_TTL:1440}" maxSize: "${CACHE_SPECS_TENANT_PROFILES_MAX_SIZE:10000}" + tenants: + timeToLiveInMinutes: "${CACHE_SPECS_TENANTS_TTL:1440}" + maxSize: "${CACHE_SPECS_TENANTS_MAX_SIZE:10000}" deviceProfiles: timeToLiveInMinutes: "${CACHE_SPECS_DEVICE_PROFILES_TTL:1440}" maxSize: "${CACHE_SPECS_DEVICE_PROFILES_MAX_SIZE:10000}" @@ -412,9 +415,21 @@ cache: edges: timeToLiveInMinutes: "${CACHE_SPECS_EDGES_TTL:1440}" maxSize: "${CACHE_SPECS_EDGES_MAX_SIZE:10000}" + repositorySettings: + timeToLiveInMinutes: "${CACHE_SPECS_REPOSITORY_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_REPOSITORY_SETTINGS_MAX_SIZE:10000}" + autoCommitSettings: + timeToLiveInMinutes: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_TTL:1440}" + maxSize: "${CACHE_SPECS_AUTO_COMMIT_SETTINGS_MAX_SIZE:10000}" twoFaVerificationCodes: timeToLiveInMinutes: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_TTL:60}" maxSize: "${CACHE_SPECS_TWO_FA_VERIFICATION_CODES_MAX_SIZE:100000}" + versionControlTask: + timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}" + maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" + +#Disable this because it is not required. +spring.data.redis.repositories.enabled: false redis: # standalone or cluster @@ -910,6 +925,9 @@ queue: tb_ota_package: - key: max.poll.records value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" @@ -1003,6 +1021,12 @@ queue: stats: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -1096,6 +1120,14 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + management: endpoints: web: diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index d908ac8d2a..2c7f9ae51a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.controller; import lombok.extern.slf4j.Slf4j; +import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.springframework.boot.test.mock.mockito.SpyBean; import org.thingsboard.server.cluster.TbClusterService; @@ -33,6 +34,7 @@ import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.model.ModelConstants; import java.util.Locale; +import java.util.Objects; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -50,73 +52,86 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { protected void testNotifyEntityAllOneTime(HasName entity, EntityId entityId, EntityId originatorId, TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { - testSendNotificationMsgToEdgeServiceOneTime(entityId, tenantId, actionType); - testLogEntityActionOneTime(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); - testPushMsgToRuleEngineOneTime(originatorId, tenantId); + int cntTime = 1; + testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); + testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); + testPushMsgToRuleEngineTime(originatorId, tenantId, cntTime); Mockito.reset(tbClusterService, auditLogService); } - protected void testNotifyEntityDeleteOneTimeMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType, Object... additionalInfo) { - testNotificationMsgToEdgeServiceNever(entityId); - testLogEntityActionOneTime(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); - testPushMsgToRuleEngineOneTime(entityId, tenantId); - testBroadcastEntityStateChangeEventOneTime(entityId, tenantId); - Mockito.reset(tbClusterService, auditLogService); - } - protected void testNotifyEntityNeverMsgToEdgeServiceOneTime(HasName entity, EntityId entityId, TenantId tenantId, ActionType actionType) { - testSendNotificationMsgToEdgeServiceOneTime(entityId, tenantId, actionType); + protected void testNotifyEntityNeverMsgToEdgeServiceOneTime(HasName entity, EntityId entityId, TenantId tenantId, + ActionType actionType) { + testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, 1); testLogEntityActionNever(entityId, entity); testPushMsgToRuleEngineNever(entityId); Mockito.reset(tbClusterService, auditLogService); } protected void testNotifyEntityOneTimeMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType, Object... additionalInfo) { + TenantId tenantId, CustomerId customerId, UserId userId, + String userName, ActionType actionType, Object... additionalInfo) { + int cntTime = 1; + testNotificationMsgToEdgeServiceNever(entityId); + testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); + testPushMsgToRuleEngineTime(originatorId, tenantId, cntTime); + Mockito.reset(tbClusterService, auditLogService); + } + + protected void testNotifyManyEntityManyTimeMsgToEdgeServiceNever(HasName entity, HasName originator, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, int cntTime, Object... additionalInfo) { + EntityId entityId = createEntityId_NULL_UUID(entity); + EntityId originatorId = createEntityId_NULL_UUID(originator); testNotificationMsgToEdgeServiceNever(entityId); - testLogEntityActionOneTime(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); - testPushMsgToRuleEngineOneTime(originatorId, tenantId); + ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); + ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); + testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + additionalInfo); + testPushMsgToRuleEngineTime(originatorId, tenantId, cntTime); Mockito.reset(tbClusterService, auditLogService); } protected void testNotifyEntityBroadcastEntityStateChangeEventOneTimeMsgToEdgeServiceNever(HasName entity, EntityId entityId, EntityId originatorId, TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { + int cntTime = 1; testNotificationMsgToEdgeServiceNever(entityId); - testLogEntityActionOneTime(entity, originatorId, tenantId, customerId, userId, userName, actionType, additionalInfo); - testPushMsgToRuleEngineOneTime(originatorId, tenantId); - testBroadcastEntityStateChangeEventOneTime(entityId, tenantId); + testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); + testPushMsgToRuleEngineTime(originatorId, tenantId, cntTime); + testBroadcastEntityStateChangeEventTime(entityId, tenantId, cntTime); Mockito.reset(tbClusterService, auditLogService); } - protected void testNotifyEntityError(HasName entity, TenantId tenantId, - UserId userId, String userName, ActionType actionType, Exception exp, - Object... additionalInfo) { + protected void testNotifyEntityEqualsOneTimeError(HasName entity, TenantId tenantId, + UserId userId, String userName, ActionType actionType, Exception exp, + Object... additionalInfo) { CustomerId customer_NULL_UUID = (CustomerId) EntityIdFactory.getByTypeAndUuid(EntityType.CUSTOMER, ModelConstants.NULL_UUID); - EntityId entity_NULL_UUID = EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(entity.getClass().toString() - .substring(entity.getClass().toString().lastIndexOf(".") + 1).toUpperCase(Locale.ENGLISH)), - ModelConstants.NULL_UUID); - testNotificationMsgToEdgeServiceNever(entity_NULL_UUID); - if (additionalInfo.length > 0) { - Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), - Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), - Mockito.argThat(argument -> - argument.getMessage().equals(exp.getMessage())), Mockito.eq(additionalInfo)); - } else { - Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customer_NULL_UUID), Mockito.eq(userId), Mockito.eq(userName), - Mockito.eq(entity_NULL_UUID), Mockito.any(entity.getClass()), Mockito.eq(actionType), - Mockito.argThat(argument -> - argument.getMessage().equals(exp.getMessage()))); - } - testPushMsgToRuleEngineNever(entity_NULL_UUID); - Mockito.reset(tbClusterService, auditLogService); + EntityId entity_originator_NULL_UUID = createEntityId_NULL_UUID(entity); + testNotificationMsgToEdgeServiceNever(entity_originator_NULL_UUID); + ArgumentMatcher matcherEntityEquals = argument -> argument.getClass().equals(entity.getClass()); + ArgumentMatcher matcherError = argument -> argument.getMessage().contains(exp.getMessage()) + & argument.getClass().equals(exp.getClass()); + testLogEntityActionErrorAdditionalInfo(matcherEntityEquals, entity_originator_NULL_UUID, tenantId, customer_NULL_UUID, userId, + userName, actionType, 1, matcherError, additionalInfo); + testPushMsgToRuleEngineNever(entity_originator_NULL_UUID); + } + + protected void testNotifyEntityIsNullOneTimeError(HasName entity, TenantId tenantId, + UserId userId, String userName, ActionType actionType, Exception exp, + Object... additionalInfo) { + CustomerId customer_NULL_UUID = (CustomerId) EntityIdFactory.getByTypeAndUuid(EntityType.CUSTOMER, ModelConstants.NULL_UUID); + EntityId entity_originator_NULL_UUID = createEntityId_NULL_UUID(entity); + testNotificationMsgToEdgeServiceNever(entity_originator_NULL_UUID); + ArgumentMatcher matcherEntityIsNull = Objects::isNull; + ArgumentMatcher matcherError = argument -> argument.getMessage().contains(exp.getMessage()) & + argument.getClass().equals(exp.getClass()); + testLogEntityActionErrorAdditionalInfo(matcherEntityIsNull, entity_originator_NULL_UUID, tenantId, customer_NULL_UUID, + userId, userName, actionType, 1, matcherError, additionalInfo); + testPushMsgToRuleEngineNever(entity_originator_NULL_UUID); } protected void testNotifyEntityNever(EntityId entityId, HasName entity) { + entityId = entityId == null ? createEntityId_NULL_UUID(entity) : entityId; testNotificationMsgToEdgeServiceNever(entityId); testLogEntityActionNever(entityId, entity); testPushMsgToRuleEngineNever(entityId); @@ -124,7 +139,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } private void testNotificationMsgToEdgeServiceNever(EntityId entityId) { - Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdgeService(Mockito.any(), + Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdge(Mockito.any(), Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any(), Mockito.any()); } @@ -139,36 +154,144 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any()); } - private void testLogEntityActionOneTime(HasName entity, EntityId originatorId, TenantId tenantId, CustomerId customerId, - UserId userId, String userName, ActionType actionType, Object... additionalInfo) { - if (additionalInfo.length == 0) { - Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), - Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), - Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull()); - } else { - String additionalInfoStr = extractParameter(String.class, 0, additionalInfo); - Mockito.verify(auditLogService, times(1)).logEntityAction(Mockito.eq(tenantId), Mockito.eq(customerId), - Mockito.eq(userId), Mockito.eq(userName), Mockito.eq(originatorId), - Mockito.eq(entity), Mockito.eq(actionType), Mockito.isNull(), Mockito.eq(additionalInfoStr)); - } + private void testPushMsgToRuleEngineTime(EntityId originatorId, TenantId tenantId, int cntTime) { + ArgumentMatcher matcherOriginatorId = cntTime == 1 ? argument -> argument.equals(originatorId) : + argument -> argument.getClass().equals(originatorId.getClass()); + Mockito.verify(tbClusterService, times(cntTime)).pushMsgToRuleEngine(Mockito.eq(tenantId), + Mockito.argThat(matcherOriginatorId), Mockito.any(TbMsg.class), Mockito.isNull()); } - private void testPushMsgToRuleEngineOneTime(EntityId originatorId, TenantId tenantId) { - Mockito.verify(tbClusterService, times(1)).pushMsgToRuleEngine(Mockito.eq(tenantId), - Mockito.eq(originatorId), Mockito.any(TbMsg.class), Mockito.isNull()); - } - - private void testSendNotificationMsgToEdgeServiceOneTime(EntityId entityId, TenantId tenantId, ActionType actionType) { - Mockito.verify(tbClusterService, times(1)).sendNotificationMsgToEdgeService(Mockito.eq(tenantId), - Mockito.isNull(), Mockito.eq(entityId), Mockito.isNull(), Mockito.isNull(), + private void testSendNotificationMsgToEdgeServiceTime(EntityId entityId, TenantId tenantId, ActionType actionType, int cntTime) { + Mockito.verify(tbClusterService, times(cntTime)).sendNotificationMsgToEdge(Mockito.eq(tenantId), + Mockito.any(), Mockito.eq(entityId), Mockito.any(), Mockito.isNull(), Mockito.eq(edgeTypeByActionType(actionType))); } - private void testBroadcastEntityStateChangeEventOneTime(EntityId entityId, TenantId tenantId) { - Mockito.verify(tbClusterService, times(1)).broadcastEntityStateChangeEvent(Mockito.eq(tenantId), + private void testBroadcastEntityStateChangeEventTime(EntityId entityId, TenantId tenantId, int cntTime) { + Mockito.verify(tbClusterService, times(cntTime)).broadcastEntityStateChangeEvent(Mockito.eq(tenantId), Mockito.any(entityId.getClass()), Mockito.any(ComponentLifecycleEvent.class)); } + private void testLogEntityAction(HasName entity, EntityId originatorId, TenantId tenantId, + CustomerId customerId, UserId userId, String userName, + ActionType actionType, int cntTime, Object... additionalInfo) { + ArgumentMatcher matcherEntityEquals = argument -> argument.equals(entity); + ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); + testLogEntityActionAdditionalInfo(matcherEntityEquals, matcherOriginatorId, tenantId, customerId, userId, userName, + actionType, cntTime, additionalInfo); + } + + private void testLogEntityActionAdditionalInfo(ArgumentMatcher matcherEntity, ArgumentMatcher matcherOriginatorId, + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, int cntTime, Object... additionalInfo) { + switch (additionalInfo.length) { + case 1: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.argThat(matcherOriginatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.isNull(), + Mockito.eq(extractParameter(String.class, 0, additionalInfo))); + break; + case 2: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.argThat(matcherOriginatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.isNull(), + Mockito.eq(extractParameter(String.class, 0, additionalInfo)), + Mockito.eq(extractParameter(String.class, 1, additionalInfo))); + break; + case 3: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.argThat(matcherOriginatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.isNull(), + Mockito.eq(extractParameter(String.class, 0, additionalInfo)), + Mockito.eq(extractParameter(String.class, 1, additionalInfo)), + Mockito.eq(extractParameter(String.class, 2, additionalInfo))); + break; + default: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.argThat(matcherOriginatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.isNull()); + } + } + + private void testLogEntityActionErrorAdditionalInfo(ArgumentMatcher matcherEntity, EntityId originatorId, TenantId tenantId, + CustomerId customerId, UserId userId, String userName, ActionType actionType, + int cntTime, ArgumentMatcher matcherError, Object... additionalInfo) { + switch (additionalInfo.length) { + case 1: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.eq(originatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.argThat(matcherError), + Mockito.eq(extractParameter(String.class, 0, additionalInfo))); + break; + case 2: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.eq(originatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.argThat(matcherError), + Mockito.eq(extractParameter(String.class, 0, additionalInfo)), + Mockito.eq(extractParameter(String.class, 1, additionalInfo))); + case 3: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.eq(originatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.argThat(matcherError), + Mockito.eq(extractParameter(String.class, 0, additionalInfo)), + Mockito.eq(extractParameter(String.class, 1, additionalInfo)), + Mockito.eq(extractParameter(String.class, 3, additionalInfo))); + break; + default: + Mockito.verify(auditLogService, times(cntTime)) + .logEntityAction(Mockito.eq(tenantId), + Mockito.eq(customerId), + Mockito.eq(userId), + Mockito.eq(userName), + Mockito.eq(originatorId), + Mockito.argThat(matcherEntity), + Mockito.eq(actionType), + Mockito.argThat(matcherError)); + } + } + private T extractParameter(Class clazz, int index, Object... additionalInfo) { T result = null; if (additionalInfo != null && additionalInfo.length > index) { @@ -179,4 +302,10 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } return result; } + + private EntityId createEntityId_NULL_UUID(HasName entity) { + return EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(entity.getClass().toString() + .substring(entity.getClass().toString().lastIndexOf(".") + 1).toUpperCase(Locale.ENGLISH)), + ModelConstants.NULL_UUID); + } } diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ee2ba371..8b17c9baf3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -237,6 +237,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { loginSysAdmin(); doDelete("/api/tenant/" + tenantId.getId().toString()) .andExpect(status().isOk()); + deleteDifferentTenant(); verifyNoTenantsLeft(); @@ -334,6 +335,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { loginSysAdmin(); doDelete("/api/tenant/" + savedDifferentTenant.getId().getId().toString()) .andExpect(status().isOk()); + savedDifferentTenant = null; } } @@ -656,7 +658,6 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected T readResponse(ResultActions result, TypeReference type) throws Exception { byte[] content = result.andReturn().getResponse().getContentAsByteArray(); - ObjectMapper mapper = new ObjectMapper(); return mapper.readerFor(type).readValue(content); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java index 0861ab34c8..0f8f2d11be 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAssetControllerTest.java @@ -22,17 +22,20 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.service.stats.DefaultRuleEngineStatisticsService; @@ -83,8 +86,14 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Asset asset = new Asset(); asset.setName("My asset"); asset.setType("default"); + + Mockito.reset(tbClusterService, auditLogService); + Asset savedAsset = doPost("/api/asset", asset, Asset.class); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedAsset, savedAsset.getId(), savedAsset.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED); + Assert.assertNotNull(savedAsset); Assert.assertNotNull(savedAsset.getId()); Assert.assertTrue(savedAsset.getCreatedTime() > 0); @@ -93,9 +102,14 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Assert.assertEquals(NULL_UUID, savedAsset.getCustomerId().getId()); Assert.assertEquals(asset.getName(), savedAsset.getName()); + Mockito.reset(tbClusterService, auditLogService); + savedAsset.setName("My new asset"); doPost("/api/asset", savedAsset, Asset.class); + testNotifyEntityAllOneTime(savedAsset, savedAsset.getId(), savedAsset.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); + Asset foundAsset = doGet("/api/asset/" + savedAsset.getId().getId().toString(), Asset.class); Assert.assertEquals(foundAsset.getName(), savedAsset.getName()); } @@ -105,13 +119,33 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Asset asset = new Asset(); asset.setName(RandomStringUtils.randomAlphabetic(300)); asset.setType("default"); - doPost("/api/asset", asset).andExpect(statusReason(containsString("length of name must be equal or less than 255"))); + + Mockito.reset(tbClusterService, auditLogService); + + String msgError = "length of name must be equal or less than 255"; + doPost("/api/asset", asset).andExpect(statusReason(containsString(msgError))); + + + testNotifyEntityEqualsOneTimeError(asset, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); + asset.setName("Normal name"); asset.setType(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/asset", asset).andExpect(statusReason(containsString("length of type must be equal or less than 255"))); + msgError = "length of type must be equal or less than 255"; + doPost("/api/asset", asset).andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeError(asset, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); + asset.setType("default"); asset.setLabel(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/asset", asset).andExpect(statusReason(containsString("length of label must be equal or less than 255"))); + msgError = "length of label must be equal or less than 255"; + doPost("/api/asset", asset).andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeError(asset, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -122,7 +156,13 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { Asset savedAsset = doPost("/api/asset", asset, Asset.class); loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/asset", savedAsset, Asset.class, status().isForbidden()); + + testNotifyEntityNever(savedAsset.getId(), savedAsset); + deleteDifferentTenant(); } @@ -140,12 +180,21 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindAssetTypesByTenantId() throws Exception { List assets = new ArrayList<>(); - for (int i = 0; i < 3; i++) { + + Mockito.reset(tbClusterService, auditLogService); + + int cntTime = 3; + for (int i = 0; i < cntTime; i++) { Asset asset = new Asset(); asset.setName("My asset B" + i); asset.setType("typeB"); assets.add(doPost("/api/asset", asset, Asset.class)); } + + testNotifyManyEntityManyTimeMsgToEdgeServiceNever(new Asset(), new Asset(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, cntTime); + for (int i = 0; i < 7; i++) { Asset asset = new Asset(); asset.setName("My asset C" + i); @@ -176,9 +225,15 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isOk()); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedAsset, savedAsset.getId(), savedAsset.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedAsset.getId().getId().toString()); + doGet("/api/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -202,9 +257,16 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { view.setType("default"); EntityView savedView = doPost("/api/entityView", view, EntityView.class); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/asset/" + savedAsset1.getId().getId().toString()) .andExpect(status().isBadRequest()); + String msgError = "Can't delete asset that has entity views"; + + testNotifyEntityIsNullOneTimeError(savedAsset1, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, new DataValidationException(msgError), savedAsset1.getId().getId().toString()); + savedView.setEntityId(savedAsset2.getId()); doPost("/api/entityView", savedView, EntityView.class); @@ -220,18 +282,32 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { public void testSaveAssetWithEmptyType() throws Exception { Asset asset = new Asset(); asset.setName("My asset"); + + Mockito.reset(tbClusterService, auditLogService); + + String msgError = "Asset type should be specified"; doPost("/api/asset", asset) .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Asset type should be specified"))); + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeError(asset, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test public void testSaveAssetWithEmptyName() throws Exception { Asset asset = new Asset(); asset.setType("default"); + + Mockito.reset(tbClusterService, auditLogService); + + String msgError = "Asset name should be specified"; doPost("/api/asset", asset) .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Asset name should be specified"))); + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeError(asset, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -245,17 +321,29 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { customer.setTitle("My customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + Mockito.reset(tbClusterService, auditLogService); + Asset assignedAsset = doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/asset/" + savedAsset.getId().getId().toString(), Asset.class); Assert.assertEquals(savedCustomer.getId(), assignedAsset.getCustomerId()); + testNotifyEntityAllOneTime(assignedAsset, assignedAsset.getId(), assignedAsset.getId(), + savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ASSIGNED_TO_CUSTOMER, savedCustomer.getId().toString(), savedCustomer.getTitle()); + Asset foundAsset = doGet("/api/asset/" + savedAsset.getId().getId().toString(), Asset.class); Assert.assertEquals(savedCustomer.getId(), foundAsset.getCustomerId()); + Mockito.reset(tbClusterService, auditLogService); + Asset unassignedAsset = doDelete("/api/customer/asset/" + savedAsset.getId().getId().toString(), Asset.class); Assert.assertEquals(ModelConstants.NULL_UUID, unassignedAsset.getCustomerId().getId()); + testNotifyEntityAllOneTime(savedAsset, savedAsset.getId(), savedAsset.getId(), + savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UNASSIGNED_FROM_CUSTOMER, savedCustomer.getId().toString(), savedCustomer.getTitle()); + foundAsset = doGet("/api/asset/" + savedAsset.getId().getId().toString(), Asset.class); Assert.assertEquals(ModelConstants.NULL_UUID, foundAsset.getCustomerId().getId()); } @@ -267,9 +355,14 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); + Mockito.reset(tbClusterService, auditLogService); + + String msgError = "Asset name should be specified"; doPost("/api/customer/" + Uuids.timeBased().toString() + "/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isNotFound()); + + testNotifyEntityNever(asset.getId(), asset); } @Test @@ -301,10 +394,14 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/asset/" + savedAsset.getId().getId().toString()) .andExpect(status().isForbidden()); + testNotifyEntityNever(savedAsset.getId(), savedAsset); + loginSysAdmin(); doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) @@ -314,7 +411,11 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { @Test public void testFindTenantAssets() throws Exception { List assets = new ArrayList<>(); - for (int i = 0; i < 178; i++) { + int cntEntity = 178; + + Mockito.reset(tbClusterService, auditLogService); + + for (int i = 0; i < cntEntity; i++) { Asset asset = new Asset(); asset.setName("Asset" + i); asset.setType("default"); @@ -332,6 +433,10 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { } } while (pageData.hasNext()); + testNotifyManyEntityManyTimeMsgToEdgeServiceNever(new Asset(), new Asset(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, cntEntity); + loadedAssets.removeIf(asset -> asset.getType().equals(DefaultRuleEngineStatisticsService.TB_SERVICE_QUEUE)); Collections.sort(assets, idComparator); @@ -731,17 +836,31 @@ public abstract class BaseAssetControllerTest extends AbstractControllerTest { asset.setType("default"); Asset savedAsset = doPost("/api/asset", asset, Asset.class); + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/edge/" + savedEdge.getId().getId().toString() + "/asset/" + savedAsset.getId().getId().toString(), Asset.class); + testNotifyEntityAllOneTime(savedAsset, savedAsset.getId(), savedAsset.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_EDGE, + savedAsset.getId().getId().toString(), savedEdge.getId().getId().toString(), edge.getName()); + + PageData pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId().toString() + "/assets?", new TypeReference>() {}, new PageLink(100)); Assert.assertEquals(1, pageData.getData().size()); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/edge/" + savedEdge.getId().getId().toString() + "/asset/" + savedAsset.getId().getId().toString(), Asset.class); + + testNotifyEntityAllOneTime(savedAsset, savedAsset.getId(), savedAsset.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UNASSIGNED_FROM_EDGE, savedAsset.getId().getId().toString(), savedEdge.getId().getId().toString(), savedEdge.getName()); + pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId().toString() + "/assets?", new TypeReference>() {}, new PageLink(100)); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java index 1f94fa4784..1e2349985f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseCustomerControllerTest.java @@ -31,6 +31,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -108,7 +109,7 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest doPost("/api/customer", savedCustomer, Customer.class); testNotifyEntityAllOneTime(savedCustomer, savedCustomer.getId(), savedCustomer.getId(), savedCustomer.getTenantId(), - savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + new CustomerId(CustomerId.NULL_UUID), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.UPDATED); Customer foundCustomer = doGet("/api/customer/" + savedCustomer.getId().getId().toString(), Customer.class); @@ -121,7 +122,6 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest @Test public void testSaveCustomerWithViolationOfValidation() throws Exception { Customer customer = new Customer(); - String validationError = "Validation error: "; customer.setTitle(RandomStringUtils.randomAlphabetic(300)); Mockito.reset(tbClusterService, auditLogService); @@ -129,49 +129,54 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest String msgError = "length of title must be equal or less than 255"; doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); + customer.setTenantId(savedTenant.getId()); + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); customer.setTitle("Normal title"); customer.setCity(RandomStringUtils.randomAlphabetic(300)); msgError = "length of city must be equal or less than 255"; doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); customer.setCity("Normal city"); customer.setCountry(RandomStringUtils.randomAlphabetic(300)); msgError = "length of country must be equal or less than 255"; doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); customer.setCountry("Ukraine"); customer.setPhone(RandomStringUtils.randomAlphabetic(300)); msgError = "length of phone must be equal or less than 255"; doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); customer.setPhone("+3892555554512"); customer.setState(RandomStringUtils.randomAlphabetic(300)); msgError = "length of state must be equal or less than 255"; doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); customer.setState("Normal state"); customer.setZip(RandomStringUtils.randomAlphabetic(300)); msgError = "length of zip or postal code must be equal or less than 255"; doPost("/api/customer", customer).andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(validationError + msgError)); - + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -210,8 +215,6 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest Assert.assertNotNull(foundCustomer); Assert.assertEquals(savedCustomer, foundCustomer); - Mockito.reset(tbClusterService, auditLogService); - doDelete("/api/customer/" + savedCustomer.getId().getId().toString()) .andExpect(status().isOk()); } @@ -246,8 +249,8 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError + "!")); + testNotifyEntityEqualsOneTimeError(customer,savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -263,16 +266,20 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityError(customer, savedTenant.getId(), - tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError + "!")); + testNotifyEntityEqualsOneTimeError(customer, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test public void testFindCustomers() throws Exception { TenantId tenantId = savedTenant.getId(); - List> futures = new ArrayList<>(135); - for (int i = 0; i < 135; i++) { + int cntEntity = 135; + + Mockito.reset(tbClusterService, auditLogService); + + List> futures = new ArrayList<>(cntEntity); + for (int i = 0; i < cntEntity; i++) { Customer customer = new Customer(); customer.setTenantId(tenantId); customer.setTitle("Customer" + i); @@ -281,6 +288,10 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest } List customers = Futures.allAsList(futures).get(TIMEOUT, TimeUnit.SECONDS); + testNotifyManyEntityManyTimeMsgToEdgeServiceNever(new Customer(), new Customer(), + tenantId, tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, cntEntity); + List loadedCustomers = new ArrayList<>(135); PageLink pageLink = new PageLink(23); PageData pageData = null; @@ -369,5 +380,4 @@ public abstract class BaseCustomerControllerTest extends AbstractControllerTest Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } - } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 143fdce0f3..f427fc7826 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -126,6 +126,16 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController Assert.assertEquals(savedDeviceProfile, foundDeviceProfile); } + @Test + public void whenGetDeviceProfileById_thenPermissionsAreChecked() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile("Device profile 1", null); + deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + + loginDifferentTenant(); + doGet("/api/deviceProfile/" + deviceProfile.getId()) + .andExpect(status().isForbidden()); + } + @Test public void testFindDeviceProfileInfoById() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); @@ -137,6 +147,16 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController Assert.assertEquals(savedDeviceProfile.getType(), foundDeviceProfileInfo.getType()); } + @Test + public void whenGetDeviceProfileInfoById_thenPermissionsAreChecked() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile("Device profile 1", null); + deviceProfile = doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + + loginDifferentTenant(); + doGet("/api/deviceProfileInfo/" + deviceProfile.getId()) + .andExpect(status().isForbidden()); + } + @Test public void testFindDefaultDeviceProfileInfo() throws Exception { DeviceProfileInfo foundDefaultDeviceProfileInfo = doGet("/api/deviceProfileInfo/default", DeviceProfileInfo.class); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java new file mode 100644 index 0000000000..65558e1955 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java @@ -0,0 +1,439 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationInfo; +import org.thingsboard.server.common.data.relation.EntityRelationsQuery; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.relation.RelationsSearchParameters; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.relation.RelationService; + +import java.util.Collections; +import java.util.List; + +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@Slf4j +public abstract class BaseEntityRelationControllerTest extends AbstractControllerTest { + + public static final String BASE_DEVICE_NAME = "Test dummy device"; + + @Autowired + RelationService relationService; + + private IdComparator idComparator; + private Tenant savedTenant; + private User tenantAdmin; + private Device mainDevice; + + @Before + public void beforeTest() throws Exception { + loginSysAdmin(); + idComparator = new IdComparator<>(); + + Tenant tenant = new Tenant(); + tenant.setTitle("Test tenant"); + + savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Assert.assertNotNull(savedTenant); + + tenantAdmin = new User(); + tenantAdmin.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin.setTenantId(savedTenant.getId()); + tenantAdmin.setEmail("tenant2@thingsboard.org"); + tenantAdmin.setFirstName("Joe"); + tenantAdmin.setLastName("Downs"); + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); + + Device device = new Device(); + device.setName("Main test device"); + device.setType("default"); + mainDevice = doPost("/api/device", device, Device.class); + } + + @After + public void afterTest() throws Exception { + loginSysAdmin(); + + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) + .andExpect(status().isOk()); + } + + @Test + public void testSaveAndFindRelation() throws Exception { + Device device = buildSimpleDevice("Test device 1"); + + EntityRelation relation = createFromRelation(mainDevice, device, "CONTAINS"); + doPost("/api/relation", relation).andExpect(status().isOk()); + + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE, + "CONTAINS", device.getUuidId(), EntityType.DEVICE + ); + + EntityRelation foundRelation = doGet(url, EntityRelation.class); + + Assert.assertNotNull("Relation is not found!", foundRelation); + Assert.assertEquals("Found relation is not equals origin!", relation, foundRelation); + } + + @Test + public void testSaveAndFindRelationsByFrom() throws Exception { + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME); + String url = String.format("/api/relations?fromId=%s&fromType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + + assertFoundList(url, numOfDevices); + } + + @Test + public void testSaveAndFindRelationsByTo() throws Exception { + final int numOfDevices = 30; + createDevicesByTo(numOfDevices, BASE_DEVICE_NAME); + String url = String.format("/api/relations?toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + + assertFoundList(url, numOfDevices); + } + + @Test + public void testSaveAndFindRelationsByFromWithRelationType() throws Exception { + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME); + + Device device = buildSimpleDevice("Unique dummy test device "); + EntityRelation relation = createFromRelation(mainDevice, device, "TEST"); + + doPost("/api/relation", relation).andExpect(status().isOk()); + String url = String.format("/api/relations?fromId=%s&fromType=%s&relationType=%s", + mainDevice.getUuidId(), EntityType.DEVICE, "TEST" + ); + + assertFoundList(url, 1); + } + + @Test + public void testSaveAndFindRelationsByToWithRelationType() throws Exception { + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME); + + Device device = buildSimpleDevice("Unique dummy test device "); + EntityRelation relation = createFromRelation(device, mainDevice, "TEST"); + + doPost("/api/relation", relation).andExpect(status().isOk()); + String url = String.format("/api/relations?toId=%s&toType=%s&relationType=%s", + mainDevice.getUuidId(), EntityType.DEVICE, "TEST" + ); + + assertFoundList(url, 1); + } + + @Test + public void testFindRelationsInfoByFrom() throws Exception { + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME); + String url = String.format("/api/relations/info?fromId=%s&fromType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + + List relationsInfos = + JacksonUtil.convertValue(doGet(url, JsonNode.class), new TypeReference>() { + }); + + Assert.assertNotNull("Relations is not found!", relationsInfos); + Assert.assertEquals("List of found relationsInfos is not equal to number of created relations!", + numOfDevices, relationsInfos.size()); + + assertRelationsInfosByFrom(relationsInfos); + } + + @Test + public void testFindRelationsInfoByTo() throws Exception { + final int numOfDevices = 30; + createDevicesByTo(numOfDevices, BASE_DEVICE_NAME); + String url = String.format("/api/relations/info?toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + + List relationsInfos = + JacksonUtil.convertValue(doGet(url, JsonNode.class), new TypeReference>() { + }); + + Assert.assertNotNull("Relations is not found!", relationsInfos); + Assert.assertEquals("List of found relationsInfos is not equal to number of created relations!", + numOfDevices, relationsInfos.size()); + + assertRelationsInfosByTo(relationsInfos); + } + + @Test + public void testDeleteRelation() throws Exception { + Device device = buildSimpleDevice("Test device 1"); + + EntityRelation relation = createFromRelation(mainDevice, device, "CONTAINS"); + doPost("/api/relation", relation).andExpect(status().isOk()); + + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE, + "CONTAINS", device.getUuidId(), EntityType.DEVICE + ); + + EntityRelation foundRelation = doGet(url, EntityRelation.class); + + Assert.assertNotNull("Relation is not found!", foundRelation); + Assert.assertEquals("Found relation is not equals origin!", relation, foundRelation); + + doDelete(url).andExpect(status().isOk()); + doGet(url).andExpect(status().is4xxClientError()); + } + + @Test + public void testDeleteRelations() throws Exception { + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME + " from"); + createDevicesByTo(numOfDevices, BASE_DEVICE_NAME + " to"); + + String urlTo = String.format("/api/relations?toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + String urlFrom = String.format("/api/relations?fromId=%s&fromType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + + assertFoundList(urlTo, numOfDevices); + assertFoundList(urlFrom, numOfDevices); + + String url = String.format("/api/relations?entityId=%s&entityType=%s", + mainDevice.getUuidId(), EntityType.DEVICE + ); + doDelete(url).andExpect(status().isOk()); + + Assert.assertTrue( + "Performed deletion of all relations but some relations were found!", + doGet(urlTo, List.class).isEmpty() + ); + Assert.assertTrue( + "Performed deletion of all relations but some relations were found!", + doGet(urlFrom, List.class).isEmpty() + ); + } + + @Test + public void testFindRelationsByFromQuery() throws Exception { + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME); + + EntityRelationsQuery query = new EntityRelationsQuery(); + query.setParameters(new RelationsSearchParameters( + mainDevice.getUuidId(), EntityType.DEVICE, + EntitySearchDirection.FROM, + RelationTypeGroup.COMMON, + 1, true + )); + query.setFilters(Collections.singletonList( + new RelationEntityTypeFilter("CONTAINS", List.of(EntityType.DEVICE)) + )); + + List relations = readResponse( + doPost("/api/relations", query).andExpect(status().isOk()), + new TypeReference>() {} + ); + + assertFoundRelations(relations, numOfDevices); + } + + @Test + public void testFindRelationsByToQuery() throws Exception { + final int numOfDevices = 30; + createDevicesByTo(numOfDevices, BASE_DEVICE_NAME); + + EntityRelationsQuery query = new EntityRelationsQuery(); + query.setParameters(new RelationsSearchParameters( + mainDevice.getUuidId(), EntityType.DEVICE, + EntitySearchDirection.TO, + RelationTypeGroup.COMMON, + 1, true + )); + query.setFilters(Collections.singletonList( + new RelationEntityTypeFilter("CONTAINS", List.of(EntityType.DEVICE)) + )); + + List relations = readResponse( + doPost("/api/relations", query).andExpect(status().isOk()), + new TypeReference>() {} + ); + + assertFoundRelations(relations, numOfDevices); + } + + @Test + public void testFindRelationsInfoByFromQuery() throws Exception{ + final int numOfDevices = 30; + createDevicesByFrom(numOfDevices, BASE_DEVICE_NAME); + + EntityRelationsQuery query = new EntityRelationsQuery(); + query.setParameters(new RelationsSearchParameters( + mainDevice.getUuidId(), EntityType.DEVICE, + EntitySearchDirection.FROM, + RelationTypeGroup.COMMON, + 1, true + )); + query.setFilters(Collections.singletonList( + new RelationEntityTypeFilter("CONTAINS", List.of(EntityType.DEVICE)) + )); + + List relationsInfo = readResponse( + doPost("/api/relations/info", query).andExpect(status().isOk()), + new TypeReference>() {} + ); + + assertRelationsInfosByFrom(relationsInfo); + } + + @Test + public void testFindRelationsInfoByToQuery() throws Exception{ + final int numOfDevices = 30; + createDevicesByTo(numOfDevices, BASE_DEVICE_NAME); + + EntityRelationsQuery query = new EntityRelationsQuery(); + query.setParameters(new RelationsSearchParameters( + mainDevice.getUuidId(), EntityType.DEVICE, + EntitySearchDirection.TO, + RelationTypeGroup.COMMON, + 1, true + )); + query.setFilters(Collections.singletonList( + new RelationEntityTypeFilter("CONTAINS", List.of(EntityType.DEVICE)) + )); + + List relationsInfo = readResponse( + doPost("/api/relations/info", query).andExpect(status().isOk()), + new TypeReference>() {} + ); + + assertRelationsInfosByTo(relationsInfo); + } + + @Test + public void testCreateRelationFromTenantToDevice() throws Exception{ + EntityRelation relation = new EntityRelation(tenantAdmin.getTenantId(), mainDevice.getId(), "CONTAINS"); + doPost("/api/relation", relation).andExpect(status().isOk()); + + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + tenantAdmin.getTenantId(), EntityType.TENANT, + "CONTAINS", mainDevice.getUuidId(), EntityType.DEVICE + ); + + EntityRelation foundRelation = doGet(url, EntityRelation.class); + + Assert.assertNotNull("Relation is not found!", foundRelation); + Assert.assertEquals("Found relation is not equals origin!", relation, foundRelation); + } + + @Test + public void testCreateRelationFromDeviceToTenant() throws Exception{ + EntityRelation relation = new EntityRelation(mainDevice.getId(), tenantAdmin.getTenantId(), "CONTAINS"); + doPost("/api/relation", relation).andExpect(status().isOk()); + + String url = String.format("/api/relation?fromId=%s&fromType=%s&relationType=%s&toId=%s&toType=%s", + mainDevice.getUuidId(), EntityType.DEVICE, + "CONTAINS", tenantAdmin.getTenantId(), EntityType.TENANT + ); + + EntityRelation foundRelation = doGet(url, EntityRelation.class); + + Assert.assertNotNull("Relation is not found!", foundRelation); + Assert.assertEquals("Found relation is not equals origin!", relation, foundRelation); + } + + private Device buildSimpleDevice(String name) throws Exception { + Device device = new Device(); + device.setName(name); + device.setType("default"); + device = doPost("/api/device", device, Device.class); + return device; + } + + private EntityRelation createFromRelation(Device mainDevice, Device device, String relationType) { + return new EntityRelation(mainDevice.getId(), device.getId(), relationType); + } + + private void createDevicesByFrom(int numOfDevices, String baseName) throws Exception { + for (int i = 0; i < numOfDevices; i++) { + Device device = buildSimpleDevice(baseName + i); + + EntityRelation relation = createFromRelation(mainDevice, device, "CONTAINS"); + doPost("/api/relation", relation).andExpect(status().isOk()); + } + } + + private void createDevicesByTo(int numOfDevices, String baseName) throws Exception { + for (int i = 0; i < numOfDevices; i++) { + Device device = buildSimpleDevice(baseName + i); + EntityRelation relation = createFromRelation(device, mainDevice, "CONTAINS"); + doPost("/api/relation", relation).andExpect(status().isOk()); + } + } + + private void assertFoundRelations(List relations, int numOfDevices) { + Assert.assertNotNull("Relations is not found!", relations); + Assert.assertEquals("List of found relations is not equal to number of created relations!", + numOfDevices, relations.size()); + } + + private void assertFoundList(String url, int numOfDevices) throws Exception { + @SuppressWarnings("unchecked") + List relations = doGet(url, List.class); + assertFoundRelations(relations, numOfDevices); + } + + private void assertRelationsInfosByFrom(List relationsInfos) { + for (EntityRelationInfo info : relationsInfos) { + Assert.assertEquals("Wrong FROM entityId!", mainDevice.getId(), info.getFrom()); + Assert.assertTrue("Wrong FROM name!", info.getToName().contains(BASE_DEVICE_NAME)); + Assert.assertEquals("Wrong relationType!", "CONTAINS", info.getType()); + } + } + + private void assertRelationsInfosByTo(List relationsInfos) { + for (EntityRelationInfo info : relationsInfos) { + Assert.assertEquals("Wrong TO entityId!", mainDevice.getId(), info.getTo()); + Assert.assertTrue("Wrong TO name!", info.getFromName().contains(BASE_DEVICE_NAME)); + Assert.assertEquals("Wrong relationType!", "CONTAINS", info.getType()); + } + } +} diff --git a/application/src/test/java/org/thingsboard/server/controller/sql/EntityRelationControllerSqlTest.java b/application/src/test/java/org/thingsboard/server/controller/sql/EntityRelationControllerSqlTest.java new file mode 100644 index 0000000000..44f4db1fd1 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/sql/EntityRelationControllerSqlTest.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller.sql; + +import org.thingsboard.server.controller.BaseEntityRelationControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; + +@DaoSqlTest +public class EntityRelationControllerSqlTest extends BaseEntityRelationControllerTest { +} diff --git a/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java index 61bb53b060..c10a9dd12d 100644 --- a/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.gson.JsonObject; import com.google.protobuf.AbstractMessage; +import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.MessageLite; import org.apache.commons.lang3.RandomStringUtils; @@ -32,7 +33,10 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.mock.web.MockMultipartFile; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Customer; @@ -42,6 +46,8 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; @@ -70,6 +76,8 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.EntityKeyValueType; @@ -107,6 +115,7 @@ import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; +import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; import org.thingsboard.server.gen.edge.v1.RpcResponseMsg; @@ -123,6 +132,7 @@ import org.thingsboard.server.gen.edge.v1.WidgetTypeUpdateMsg; import org.thingsboard.server.gen.edge.v1.WidgetsBundleUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -133,18 +143,21 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; @TestPropertySource(properties = { "edges.enabled=true", }) abstract public class BaseEdgeTest extends AbstractControllerTest { - private static final String CUSTOM_DEVICE_PROFILE_NAME = "Thermostat"; + private static final String THERMOSTAT_DEVICE_PROFILE_NAME = "Thermostat"; private Tenant savedTenant; private TenantId tenantId; private User tenantAdmin; + private DeviceProfile thermostatDeviceProfile; + private EdgeImitator edgeImitator; private Edge edge; @@ -204,12 +217,12 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { MqttDeviceProfileTransportConfiguration transportConfiguration = new MqttDeviceProfileTransportConfiguration(); transportConfiguration.setTransportPayloadTypeConfiguration(new JsonTransportPayloadConfiguration()); - DeviceProfile deviceProfile = this.createDeviceProfile(CUSTOM_DEVICE_PROFILE_NAME, transportConfiguration); + thermostatDeviceProfile = this.createDeviceProfile(THERMOSTAT_DEVICE_PROFILE_NAME, transportConfiguration); - extendDeviceProfileData(deviceProfile); - doPost("/api/deviceProfile", deviceProfile, DeviceProfile.class); + extendDeviceProfileData(thermostatDeviceProfile); + thermostatDeviceProfile = doPost("/api/deviceProfile", thermostatDeviceProfile, DeviceProfile.class); - Device savedDevice = saveDevice("Edge Device 1", CUSTOM_DEVICE_PROFILE_NAME); + Device savedDevice = saveDevice("Edge Device 1", THERMOSTAT_DEVICE_PROFILE_NAME); doPost("/api/edge/" + edge.getUuidId() + "/device/" + savedDevice.getUuidId(), Device.class); @@ -269,7 +282,7 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { List deviceProfileUpdateMsgList = edgeImitator.findAllMessagesByType(DeviceProfileUpdateMsg.class); Assert.assertEquals(3, deviceProfileUpdateMsgList.size()); Optional deviceProfileUpdateMsgOpt = - deviceProfileUpdateMsgList.stream().filter(dfum -> CUSTOM_DEVICE_PROFILE_NAME.equals(dfum.getName())).findAny(); + deviceProfileUpdateMsgList.stream().filter(dfum -> THERMOSTAT_DEVICE_PROFILE_NAME.equals(dfum.getName())).findAny(); Assert.assertTrue(deviceProfileUpdateMsgOpt.isPresent()); DeviceProfileUpdateMsg deviceProfileUpdateMsg = deviceProfileUpdateMsgOpt.get(); Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, deviceProfileUpdateMsg.getMsgType()); @@ -1339,17 +1352,17 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { String timeseriesKey = "key"; String timeseriesValue = "25"; data.addProperty(timeseriesKey, timeseriesValue); - UplinkMsg.Builder uplinkMsgBuilder1 = UplinkMsg.newBuilder(); + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); EntityDataProto.Builder entityDataBuilder = EntityDataProto.newBuilder(); entityDataBuilder.setPostTelemetryMsg(JsonConverter.convertToTelemetryProto(data, System.currentTimeMillis())); entityDataBuilder.setEntityType(device.getId().getEntityType().name()); entityDataBuilder.setEntityIdMSB(device.getUuidId().getMostSignificantBits()); entityDataBuilder.setEntityIdLSB(device.getUuidId().getLeastSignificantBits()); testAutoGeneratedCodeByProtobuf(entityDataBuilder); - uplinkMsgBuilder1.addEntityData(entityDataBuilder.build()); + uplinkMsgBuilder.addEntityData(entityDataBuilder.build()); - testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder1); - edgeImitator.sendUplinkMsg(uplinkMsgBuilder1.build()); + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); JsonObject attributesData = new JsonObject(); String attributesKey = "test_attr"; @@ -1381,9 +1394,24 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/" + DataConstants.SERVER_SCOPE; List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() {}); - Assert.assertEquals(2, attributes.size()); - var result = attributes.stream().filter(kv -> kv.get("key").equals(attributesKey)).filter(kv -> kv.get("value").equals(attributesValue)).findFirst(); - Assert.assertTrue(result.isPresent()); + + Assert.assertEquals(3, attributes.size()); + + Optional> activeAttributeOpt = getAttributeByKey("active", attributes); + Assert.assertTrue(activeAttributeOpt.isPresent()); + Map activeAttribute = activeAttributeOpt.get(); + Assert.assertEquals("true", activeAttribute.get("value")); + + Optional> customAttributeOpt = getAttributeByKey(attributesKey, attributes); + Assert.assertTrue(customAttributeOpt.isPresent()); + Map customAttribute = customAttributeOpt.get(); + Assert.assertEquals(attributesValue, customAttribute.get("value")); + + doDelete("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/SERVER_SCOPE?keys=" + attributesKey, String.class); + } + + private Optional> getAttributeByKey(String key, List> attributes) { + return attributes.stream().filter(kv -> kv.get("key").equals(key)).findFirst(); } private Map>> loadDeviceTimeseries(Device device, String timeseriesKey) throws Exception { @@ -1643,6 +1671,110 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { Assert.assertTrue("Expected key and value must be found", found); } + @Test + public void testOtaPackages_usesUrl() throws Exception { + // 1 + SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); + firmwareInfo.setDeviceProfileId(thermostatDeviceProfile.getId()); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle("My firmware #1"); + firmwareInfo.setVersion("v1.0"); + firmwareInfo.setTag("My firmware #1 v1.0"); + firmwareInfo.setUsesUrl(true); + firmwareInfo.setUrl("http://localhost:8080/v1/package"); + firmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode()); + + edgeImitator.expectMessageAmount(1); + OtaPackageInfo savedFirmwareInfo = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof OtaPackageUpdateMsg); + OtaPackageUpdateMsg otaPackageUpdateMsg = (OtaPackageUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, otaPackageUpdateMsg.getMsgType()); + Assert.assertEquals(savedFirmwareInfo.getUuidId().getMostSignificantBits(), otaPackageUpdateMsg.getIdMSB()); + Assert.assertEquals(savedFirmwareInfo.getUuidId().getLeastSignificantBits(), otaPackageUpdateMsg.getIdLSB()); + Assert.assertEquals(thermostatDeviceProfile.getUuidId().getMostSignificantBits(), otaPackageUpdateMsg.getDeviceProfileIdMSB()); + Assert.assertEquals(thermostatDeviceProfile.getUuidId().getLeastSignificantBits(), otaPackageUpdateMsg.getDeviceProfileIdLSB()); + Assert.assertEquals(FIRMWARE, OtaPackageType.valueOf(otaPackageUpdateMsg.getType())); + Assert.assertEquals("My firmware #1", otaPackageUpdateMsg.getTitle()); + Assert.assertEquals("v1.0", otaPackageUpdateMsg.getVersion()); + Assert.assertEquals("My firmware #1 v1.0", otaPackageUpdateMsg.getTag()); + Assert.assertEquals("http://localhost:8080/v1/package", otaPackageUpdateMsg.getUrl()); + Assert.assertFalse(otaPackageUpdateMsg.hasData()); + Assert.assertFalse(otaPackageUpdateMsg.hasFileName()); + Assert.assertFalse(otaPackageUpdateMsg.hasContentType()); + Assert.assertFalse(otaPackageUpdateMsg.hasChecksumAlgorithm()); + Assert.assertFalse(otaPackageUpdateMsg.hasChecksum()); + Assert.assertFalse(otaPackageUpdateMsg.hasDataSize()); + + // 2 + edgeImitator.expectMessageAmount(1); + doDelete("/api/otaPackage/" + savedFirmwareInfo.getUuidId()) + .andExpect(status().isOk()); + Assert.assertTrue(edgeImitator.waitForMessages()); + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof OtaPackageUpdateMsg); + otaPackageUpdateMsg = (OtaPackageUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, otaPackageUpdateMsg.getMsgType()); + Assert.assertEquals(otaPackageUpdateMsg.getIdMSB(), savedFirmwareInfo.getUuidId().getMostSignificantBits()); + Assert.assertEquals(otaPackageUpdateMsg.getIdLSB(), savedFirmwareInfo.getUuidId().getLeastSignificantBits()); + } + + @Test + public void testOtaPackages_hasData() throws Exception { + // 1 + SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); + firmwareInfo.setDeviceProfileId(thermostatDeviceProfile.getId()); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle("My firmware #2"); + firmwareInfo.setVersion("v2.0"); + firmwareInfo.setTag("My firmware #2 v2.0"); + firmwareInfo.setUsesUrl(false); + firmwareInfo.setHasData(false); + firmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode()); + + edgeImitator.expectMessageAmount(1); + + OtaPackageInfo savedFirmwareInfo = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + MockMultipartFile testData = new MockMultipartFile("file", "firmware.bin", "image/png", ByteBuffer.wrap(new byte[]{1, 3, 5}).array()); + savedFirmwareInfo = saveData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksumAlgorithm={checksumAlgorithm}", testData, ChecksumAlgorithm.SHA256.name()); + + Assert.assertTrue(edgeImitator.waitForMessages()); + + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof OtaPackageUpdateMsg); + OtaPackageUpdateMsg otaPackageUpdateMsg = (OtaPackageUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, otaPackageUpdateMsg.getMsgType()); + Assert.assertEquals(savedFirmwareInfo.getUuidId().getMostSignificantBits(), otaPackageUpdateMsg.getIdMSB()); + Assert.assertEquals(savedFirmwareInfo.getUuidId().getLeastSignificantBits(), otaPackageUpdateMsg.getIdLSB()); + Assert.assertEquals(thermostatDeviceProfile.getUuidId().getMostSignificantBits(), otaPackageUpdateMsg.getDeviceProfileIdMSB()); + Assert.assertEquals(thermostatDeviceProfile.getUuidId().getLeastSignificantBits(), otaPackageUpdateMsg.getDeviceProfileIdLSB()); + Assert.assertEquals(FIRMWARE, OtaPackageType.valueOf(otaPackageUpdateMsg.getType())); + Assert.assertEquals("My firmware #2", otaPackageUpdateMsg.getTitle()); + Assert.assertEquals("v2.0", otaPackageUpdateMsg.getVersion()); + Assert.assertEquals("My firmware #2 v2.0", otaPackageUpdateMsg.getTag()); + Assert.assertFalse(otaPackageUpdateMsg.hasUrl()); + Assert.assertEquals("firmware.bin", otaPackageUpdateMsg.getFileName()); + Assert.assertEquals("image/png", otaPackageUpdateMsg.getContentType()); + Assert.assertEquals(ChecksumAlgorithm.SHA256.name(), otaPackageUpdateMsg.getChecksumAlgorithm()); + Assert.assertEquals("62467691cf583d4fa78b18fafaf9801f505e0ef03baf0603fd4b0cd004cd1e75", otaPackageUpdateMsg.getChecksum()); + Assert.assertEquals(3L, otaPackageUpdateMsg.getDataSize()); + Assert.assertEquals(ByteString.copyFrom(new byte[]{1, 3, 5}), otaPackageUpdateMsg.getData()); + + // 2 + edgeImitator.expectMessageAmount(1); + doDelete("/api/otaPackage/" + savedFirmwareInfo.getUuidId()) + .andExpect(status().isOk()); + Assert.assertTrue(edgeImitator.waitForMessages()); + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof OtaPackageUpdateMsg); + otaPackageUpdateMsg = (OtaPackageUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, otaPackageUpdateMsg.getMsgType()); + Assert.assertEquals(otaPackageUpdateMsg.getIdMSB(), savedFirmwareInfo.getUuidId().getMostSignificantBits()); + Assert.assertEquals(otaPackageUpdateMsg.getIdLSB(), savedFirmwareInfo.getUuidId().getLeastSignificantBits()); + } + // Utility methods private Device saveDeviceOnCloudAndVerifyDeliveryToEdge() throws Exception { @@ -1723,4 +1855,12 @@ abstract public class BaseEdgeTest extends AbstractControllerTest { Assert.assertEquals(source, target); Assert.assertEquals(source.hashCode(), target.hashCode()); } + + private OtaPackageInfo saveData(String urlTemplate, MockMultipartFile content, String... params) throws Exception { + MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params); + postRequest.file(content); + setJwtToken(postRequest); + return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackageInfo.class); + } + } diff --git a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java index cfb652dd9f..ac05ee3588 100644 --- a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java +++ b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java @@ -41,6 +41,7 @@ import org.thingsboard.server.gen.edge.v1.DownlinkResponseMsg; import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.edge.v1.EntityDataProto; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; +import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainMetadataUpdateMsg; import org.thingsboard.server.gen.edge.v1.RuleChainUpdateMsg; @@ -277,6 +278,11 @@ public class EdgeImitator { result.add(saveDownlinkMsg(deviceCredentialsRequestMsg)); } } + if (downlinkMsg.getOtaPackageUpdateMsgCount() > 0) { + for (OtaPackageUpdateMsg otaPackageUpdateMsg : downlinkMsg.getOtaPackageUpdateMsgList()) { + result.add(saveDownlinkMsg(otaPackageUpdateMsg)); + } + } return Futures.allAsList(result); } diff --git a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java index cea08003bf..9df6c16111 100644 --- a/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java @@ -71,6 +71,8 @@ public class HashPartitionServiceTest { queueRoutingInfoService); ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core"); ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10); + ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); + ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() .setServiceId("tb-core-0") diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java new file mode 100644 index 0000000000..e516d5e131 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -0,0 +1,452 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.HasTenantId; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.dashboard.DashboardService; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; +import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +public abstract class BaseExportImportServiceTest extends AbstractControllerTest { + + @Autowired + protected EntitiesExportImportService exportImportService; + @Autowired + protected DeviceService deviceService; + @Autowired + protected OtaPackageService otaPackageService; + @Autowired + protected DeviceProfileService deviceProfileService; + @Autowired + protected AssetService assetService; + @Autowired + protected CustomerService customerService; + @Autowired + protected RuleChainService ruleChainService; + @Autowired + protected DashboardService dashboardService; + @Autowired + protected RelationService relationService; + @Autowired + protected TenantService tenantService; + @Autowired + protected EntityViewService entityViewService; + + protected TenantId tenantId1; + protected User tenantAdmin1; + + protected TenantId tenantId2; + protected User tenantAdmin2; + + @Before + public void beforeEach() throws Exception { + loginSysAdmin(); + Tenant tenant1 = new Tenant(); + tenant1.setTitle("Tenant 1"); + tenant1.setEmail("tenant1@thingsboard.org"); + this.tenantId1 = tenantService.saveTenant(tenant1).getId(); + User tenantAdmin1 = new User(); + tenantAdmin1.setTenantId(tenantId1); + tenantAdmin1.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin1.setEmail("tenant1-admin@thingsboard.org"); + this.tenantAdmin1 = createUser(tenantAdmin1, "12345678"); + Tenant tenant2 = new Tenant(); + tenant2.setTitle("Tenant 2"); + tenant2.setEmail("tenant2@thingsboard.org"); + this.tenantId2 = tenantService.saveTenant(tenant2).getId(); + User tenantAdmin2 = new User(); + tenantAdmin2.setTenantId(tenantId2); + tenantAdmin2.setAuthority(Authority.TENANT_ADMIN); + tenantAdmin2.setEmail("tenant2-admin@thingsboard.org"); + this.tenantAdmin2 = createUser(tenantAdmin2, "12345678"); + } + + @After + public void afterEach() { + tenantService.deleteTenant(tenantId1); + tenantService.deleteTenant(tenantId2); + } + + protected Device createDevice(TenantId tenantId, CustomerId customerId, DeviceProfileId deviceProfileId, String name) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setCustomerId(customerId); + device.setName(name); + device.setLabel("lbl"); + device.setDeviceProfileId(deviceProfileId); + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(new DefaultDeviceTransportConfiguration()); + device.setDeviceData(deviceData); + return deviceService.saveDevice(device); + } + + protected OtaPackage createOtaPackage(TenantId tenantId, DeviceProfileId deviceProfileId, OtaPackageType type) { + OtaPackage otaPackage = new OtaPackage(); + otaPackage.setTenantId(tenantId); + otaPackage.setDeviceProfileId(deviceProfileId); + otaPackage.setType(type); + otaPackage.setTitle("My " + type); + otaPackage.setVersion("v1.0"); + otaPackage.setFileName("filename.txt"); + otaPackage.setContentType("text/plain"); + otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); + otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); + otaPackage.setDataSize(1L); + otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1})); + return otaPackageService.saveOtaPackage(otaPackage); + } + + protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) { + assertThat(importedDevice.getName()).isEqualTo(initialDevice.getName()); + assertThat(importedDevice.getType()).isEqualTo(initialDevice.getType()); + assertThat(importedDevice.getDeviceData()).isEqualTo(initialDevice.getDeviceData()); + assertThat(importedDevice.getLabel()).isEqualTo(initialDevice.getLabel()); + } + + protected DeviceProfile createDeviceProfile(TenantId tenantId, RuleChainId defaultRuleChainId, DashboardId defaultDashboardId, String name) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setName(name); + deviceProfile.setDescription("dscrptn"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + deviceProfile.setTransportType(DeviceTransportType.DEFAULT); + deviceProfile.setDefaultRuleChainId(defaultRuleChainId); + deviceProfile.setDefaultDashboardId(defaultDashboardId); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + profileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + protected void checkImportedDeviceProfileData(DeviceProfile initialProfile, DeviceProfile importedProfile) { + assertThat(initialProfile.getName()).isEqualTo(importedProfile.getName()); + assertThat(initialProfile.getType()).isEqualTo(importedProfile.getType()); + assertThat(initialProfile.getTransportType()).isEqualTo(importedProfile.getTransportType()); + assertThat(initialProfile.getProfileData()).isEqualTo(importedProfile.getProfileData()); + assertThat(initialProfile.getDescription()).isEqualTo(importedProfile.getDescription()); + } + + protected Asset createAsset(TenantId tenantId, CustomerId customerId, String type, String name) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setCustomerId(customerId); + asset.setType(type); + asset.setName(name); + asset.setLabel("lbl"); + asset.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return assetService.saveAsset(asset); + } + + protected void checkImportedAssetData(Asset initialAsset, Asset importedAsset) { + assertThat(importedAsset.getName()).isEqualTo(initialAsset.getName()); + assertThat(importedAsset.getType()).isEqualTo(initialAsset.getType()); + assertThat(importedAsset.getLabel()).isEqualTo(initialAsset.getLabel()); + assertThat(importedAsset.getAdditionalInfo()).isEqualTo(initialAsset.getAdditionalInfo()); + } + + protected Customer createCustomer(TenantId tenantId, String name) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(name); + customer.setCountry("ua"); + customer.setAddress("abb"); + customer.setEmail("ccc@aa.org"); + customer.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + return customerService.saveCustomer(customer); + } + + protected void checkImportedCustomerData(Customer initialCustomer, Customer importedCustomer) { + assertThat(importedCustomer.getTitle()).isEqualTo(initialCustomer.getTitle()); + assertThat(importedCustomer.getCountry()).isEqualTo(initialCustomer.getCountry()); + assertThat(importedCustomer.getAddress()).isEqualTo(initialCustomer.getAddress()); + assertThat(importedCustomer.getEmail()).isEqualTo(initialCustomer.getEmail()); + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenantId); + dashboard.setTitle(name); + dashboard.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + dashboard.setImage("abvregewrg"); + dashboard.setMobileHide(true); + dashboard = dashboardService.saveDashboard(dashboard); + if (customerId != null) { + dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId); + return dashboardService.findDashboardById(tenantId, dashboard.getId()); + } + return dashboard; + } + + protected Dashboard createDashboard(TenantId tenantId, CustomerId customerId, String name, AssetId assetForEntityAlias) { + Dashboard dashboard = createDashboard(tenantId, customerId, name); + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + assetForEntityAlias.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + return dashboardService.saveDashboard(dashboard); + } + + protected void checkImportedDashboardData(Dashboard initialDashboard, Dashboard importedDashboard) { + assertThat(importedDashboard.getTitle()).isEqualTo(initialDashboard.getTitle()); + assertThat(importedDashboard.getConfiguration()).isEqualTo(initialDashboard.getConfiguration()); + assertThat(importedDashboard.getImage()).isEqualTo(initialDashboard.getImage()); + assertThat(importedDashboard.isMobileHide()).isEqualTo(initialDashboard.isMobileHide()); + if (initialDashboard.getAssignedCustomers() != null) { + assertThat(importedDashboard.getAssignedCustomers()).containsAll(initialDashboard.getAssignedCustomers()); + } + } + protected RuleChain createRuleChain(TenantId tenantId, String name, EntityId originatorId) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Generator 1"); + ruleNode1.setType(TbMsgGeneratorNode.class.getName()); + ruleNode1.setDebugMode(true); + TbMsgGeneratorNodeConfiguration configuration1 = new TbMsgGeneratorNodeConfiguration(); + configuration1.setOriginatorType(originatorId.getEntityType()); + configuration1.setOriginatorId(originatorId.getId().toString()); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + + protected RuleChain createRuleChain(TenantId tenantId, String name) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName(name); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChain.getId()); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(mapper.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(mapper.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, "Success"); + ruleChainService.saveRuleChainMetaData(tenantId, metaData); + + return ruleChainService.findRuleChainById(tenantId, ruleChain.getId()); + } + + protected void checkImportedRuleChainData(RuleChain initialRuleChain, RuleChainMetaData initialMetaData, RuleChain importedRuleChain, RuleChainMetaData importedMetaData) { + assertThat(importedRuleChain.getType()).isEqualTo(initialRuleChain.getType()); + assertThat(importedRuleChain.getName()).isEqualTo(initialRuleChain.getName()); + assertThat(importedRuleChain.isDebugMode()).isEqualTo(initialRuleChain.isDebugMode()); + assertThat(importedRuleChain.getConfiguration()).isEqualTo(initialRuleChain.getConfiguration()); + + assertThat(importedMetaData.getConnections()).isEqualTo(initialMetaData.getConnections()); + assertThat(importedMetaData.getFirstNodeIndex()).isEqualTo(initialMetaData.getFirstNodeIndex()); + for (int i = 0; i < initialMetaData.getNodes().size(); i++) { + RuleNode initialNode = initialMetaData.getNodes().get(i); + RuleNode importedNode = importedMetaData.getNodes().get(i); + assertThat(importedNode.getRuleChainId()).isEqualTo(importedRuleChain.getId()); + assertThat(importedNode.getName()).isEqualTo(initialNode.getName()); + assertThat(importedNode.getType()).isEqualTo(initialNode.getType()); + assertThat(importedNode.getConfiguration()).isEqualTo(initialNode.getConfiguration()); + assertThat(importedNode.getAdditionalInfo()).isEqualTo(initialNode.getAdditionalInfo()); + } + } + + protected EntityView createEntityView(TenantId tenantId, CustomerId customerId, EntityId entityId, String name) { + EntityView entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setEntityId(entityId); + entityView.setCustomerId(customerId); + entityView.setName(name); + entityView.setType("A"); + return entityViewService.saveEntityView(entityView); + } + + protected EntityRelation createRelation(EntityId from, EntityId to) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(from); + relation.setTo(to); + relation.setType(EntityRelation.MANAGES_TYPE); + relation.setAdditionalInfo(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relationService.saveRelation(TenantId.SYS_TENANT_ID, relation); + return relation; + } + + protected & HasTenantId> void checkImportedEntity(TenantId tenantId1, E initialEntity, TenantId tenantId2, E importedEntity) { + assertThat(initialEntity.getTenantId()).isEqualTo(tenantId1); + assertThat(importedEntity.getTenantId()).isEqualTo(tenantId2); + + assertThat(importedEntity.getExternalId()).isEqualTo(initialEntity.getId()); + + boolean sameTenant = tenantId1.equals(tenantId2); + if (!sameTenant) { + assertThat(importedEntity.getId()).isNotEqualTo(initialEntity.getId()); + } else { + assertThat(importedEntity.getId()).isEqualTo(initialEntity.getId()); + } + } + + + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId) throws Exception { + return exportEntity(user, entityId, EntityExportSettings.builder() + .exportCredentials(true) + .build()); + } + + protected , I extends EntityId> EntityExportData exportEntity(User user, I entityId, EntityExportSettings exportSettings) throws Exception { + return exportImportService.exportEntity(new SimpleEntitiesExportCtx(getSecurityUser(user), null, null, exportSettings), entityId); + } + + protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData) throws Exception { + return importEntity(user, exportData, EntityImportSettings.builder() + .saveCredentials(true) + .build()); + } + + protected , I extends EntityId> EntityImportResult importEntity(User user, EntityExportData exportData, EntityImportSettings importSettings) throws Exception { + EntitiesImportCtx ctx = new EntitiesImportCtx(UUID.randomUUID(), getSecurityUser(user), null, importSettings); + ctx.setFinalImportAttempt(true); + exportData = JacksonUtil.treeToValue(JacksonUtil.valueToTree(exportData), EntityExportData.class); + EntityImportResult importResult = exportImportService.importEntity(ctx, exportData); + exportImportService.saveReferencesAndRelations(ctx); + for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) { + throwingRunnable.run(); + } + return importResult; + } + + protected SecurityUser getSecurityUser(User user) { + return new SecurityUser(user, true, new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail())); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java new file mode 100644 index 0000000000..8cf57e8e37 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/ExportImportServiceSqlTest.java @@ -0,0 +1,585 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.ie; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.Streams; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; +import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.audit.ActionType; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.OtaPackageType; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.dao.device.DeviceProfileDao; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.service.action.EntityActionService; +import org.thingsboard.server.service.ota.OtaPackageStateService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.verify; + +@DaoSqlTest +public class ExportImportServiceSqlTest extends BaseExportImportServiceTest { + + @Autowired + private DeviceCredentialsService deviceCredentialsService; + @SpyBean + private EntityActionService entityActionService; + @SpyBean + private OtaPackageStateService otaPackageStateService; + + @Test + public void testExportImportAsset_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, asset.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, asset, tenantId2, importResult.getSavedEntity()); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_sameTenant() throws Exception { + Asset asset = createAsset(tenantId1, null, "AB", "Asset v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, asset.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, asset, tenantId1, importResult.getSavedEntity()); + checkImportedAssetData(asset, importResult.getSavedEntity()); + } + + @Test + public void testExportImportAsset_sameTenant_withCustomer() throws Exception { + Customer customer = createCustomer(tenantId1, "My customer"); + Asset asset = createAsset(tenantId1, customer.getId(), "AB", "My asset"); + + Asset importedAsset = importEntity(tenantAdmin1, this.exportEntity(tenantAdmin1, asset.getId())).getSavedEntity(); + assertThat(importedAsset.getCustomerId()).isEqualTo(asset.getCustomerId()); + } + + + @Test + public void testExportImportCustomer_betweenTenants() throws Exception { + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, customer.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, customer, tenantId2, importResult.getSavedEntity()); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + @Test + public void testExportImportCustomer_sameTenant() throws Exception { + Customer customer = createCustomer(tenantAdmin1.getTenantId(), "Customer v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, customer.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, customer, tenantId1, importResult.getSavedEntity()); + checkImportedCustomerData(customer, importResult.getSavedEntity()); + } + + + @Test + public void testExportImportDeviceWithProfile_betweenTenants() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile of tenant 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device of tenant 1"); + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData profileExportData = exportEntity(tenantAdmin1, deviceProfile.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId()); + DeviceCredentials exportedCredentials = ((DeviceExportData) deviceExportData).getCredentials(); + exportedCredentials.setCredentialsId(credentials.getCredentialsId() + "a"); + + EntityImportResult profileImportResult = importEntity(tenantAdmin2, profileExportData); + checkImportedEntity(tenantId1, deviceProfile, tenantId2, profileImportResult.getSavedEntity()); + checkImportedDeviceProfileData(deviceProfile, profileImportResult.getSavedEntity()); + + EntityImportResult deviceImportResult = importEntity(tenantAdmin2, deviceExportData); + Device importedDevice = deviceImportResult.getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, deviceImportResult.getSavedEntity()); + checkImportedDeviceData(device, importedDevice); + + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(profileImportResult.getSavedEntity().getId()); + + DeviceCredentials importedCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId2, importedDevice.getId()); + assertThat(importedCredentials.getId()).isNotEqualTo(credentials.getId()); + assertThat(importedCredentials.getCredentialsId()).isEqualTo(exportedCredentials.getCredentialsId()); + assertThat(importedCredentials.getCredentialsValue()).isEqualTo(credentials.getCredentialsValue()); + assertThat(importedCredentials.getCredentialsType()).isEqualTo(credentials.getCredentialsType()); + } + + @Test + public void testExportImportDevice_sameTenant() throws Exception { + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, null, null, "Device profile v1.0"); + OtaPackage firmware = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.FIRMWARE); + OtaPackage software = createOtaPackage(tenantId1, deviceProfile.getId(), OtaPackageType.SOFTWARE); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device v1.0"); + device.setFirmwareId(firmware.getId()); + device.setSoftwareId(software.getId()); + device = deviceService.saveDevice(device); + + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, deviceExportData); + Device importedDevice = importResult.getSavedEntity(); + + checkImportedEntity(tenantId1, device, tenantId1, importResult.getSavedEntity()); + assertThat(importedDevice.getDeviceProfileId()).isEqualTo(device.getDeviceProfileId()); + assertThat(deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId1, device.getId())).isEqualTo(credentials); + assertThat(importedDevice.getFirmwareId()).isEqualTo(firmware.getId()); + assertThat(importedDevice.getSoftwareId()).isEqualTo(software.getId()); + } + + + @Test + public void testExportImportDashboard_betweenTenants() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + checkImportedEntity(tenantId1, dashboard, tenantId2, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } + + @Test + public void testExportImportDashboard_sameTenant() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard v1.0"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + checkImportedEntity(tenantId1, dashboard, tenantId1, importResult.getSavedEntity()); + checkImportedDashboardData(dashboard, importResult.getSavedEntity()); + } + + @Test + public void testExportImportDashboard_betweenTenants_withCustomer_updated() throws Exception { + Dashboard dashboard = createDashboard(tenantAdmin1.getTenantId(), null, "Dashboard of tenant 1"); + EntityExportData exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Dashboard importedDashboard = importEntity(tenantAdmin2, exportData).getSavedEntity(); + checkImportedEntity(tenantId1, dashboard, tenantId2, importedDashboard); + + Customer customer = createCustomer(tenantId1, "Customer 1"); + EntityExportData customerExportData = exportEntity(tenantAdmin1, customer.getId()); + dashboardService.assignDashboardToCustomer(tenantId1, dashboard.getId(), customer.getId()); + exportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Customer importedCustomer = importEntity(tenantAdmin2, customerExportData).getSavedEntity(); + importedDashboard = importEntity(tenantAdmin2, exportData).getSavedEntity(); + assertThat(importedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(customerInfo -> { + assertThat(customerInfo.getCustomerId()).isEqualTo(importedCustomer.getId()); + }); + } + + @Test + public void testExportImportDashboard_betweenTenants_withEntityAliases() throws Exception { + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + + String entityAliases = "{\n" + + "\t\"23c4185d-1497-9457-30b2-6d91e69a5b2c\": {\n" + + "\t\t\"alias\": \"assets\",\n" + + "\t\t\"filter\": {\n" + + "\t\t\t\"entityList\": [\n" + + "\t\t\t\t\"" + asset1.getId().toString() + "\",\n" + + "\t\t\t\t\"" + asset2.getId().toString() + "\"\n" + + "\t\t\t],\n" + + "\t\t\t\"entityType\": \"ASSET\",\n" + + "\t\t\t\"resolveMultiple\": true,\n" + + "\t\t\t\"type\": \"entityList\"\n" + + "\t\t},\n" + + "\t\t\"id\": \"23c4185d-1497-9457-30b2-6d91e69a5b2c\"\n" + + "\t}\n" + + "}"; + ObjectNode dashboardConfiguration = JacksonUtil.newObjectNode(); + dashboardConfiguration.set("entityAliases", JacksonUtil.toJsonNode(entityAliases)); + dashboardConfiguration.set("description", new TextNode("hallo")); + dashboard.setConfiguration(dashboardConfiguration); + dashboard = dashboardService.saveDashboard(dashboard); + + EntityExportData asset1ExportData = exportEntity(tenantAdmin1, asset1.getId()); + EntityExportData asset2ExportData = exportEntity(tenantAdmin1, asset2.getId()); + EntityExportData dashboardExportData = exportEntity(tenantAdmin1, dashboard.getId()); + + Asset importedAsset1 = importEntity(tenantAdmin2, asset1ExportData).getSavedEntity(); + Asset importedAsset2 = importEntity(tenantAdmin2, asset2ExportData).getSavedEntity(); + Dashboard importedDashboard = importEntity(tenantAdmin2, dashboardExportData).getSavedEntity(); + + Set entityAliasEntitiesIds = Streams.stream(importedDashboard.getConfiguration() + .get("entityAliases").elements().next().get("filter").get("entityList").elements()) + .map(JsonNode::asText).collect(Collectors.toSet()); + assertThat(entityAliasEntitiesIds).doesNotContain(asset1.getId().toString(), asset2.getId().toString()); + assertThat(entityAliasEntitiesIds).contains(importedAsset1.getId().toString(), importedAsset2.getId().toString()); + } + + + @Test + public void testExportImportRuleChain_betweenTenants() throws Exception { + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain of tenant 1"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + EntityExportData exportData = exportEntity(tenantAdmin1, ruleChain.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin2, exportData); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId2, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId2, importResult.getSavedEntity()); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + @Test + public void testExportImportRuleChain_sameTenant() throws Exception { + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain v1.0"); + RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId1, ruleChain.getId()); + EntityExportData exportData = exportEntity(tenantAdmin1, ruleChain.getId()); + + EntityImportResult importResult = importEntity(tenantAdmin1, exportData); + RuleChain importedRuleChain = importResult.getSavedEntity(); + RuleChainMetaData importedMetaData = ruleChainService.loadRuleChainMetaData(tenantId1, importedRuleChain.getId()); + + checkImportedEntity(tenantId1, ruleChain, tenantId1, importResult.getSavedEntity()); + checkImportedRuleChainData(ruleChain, metaData, importedRuleChain, importedMetaData); + } + + + @Test + public void testExportImportWithInboundRelations_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId()); + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .exportCredentials(false) + .build()); + + assertThat(deviceExportData.getRelations()).size().isOne(); + assertThat(deviceExportData.getRelations().get(0)).matches(entityRelation -> { + return entityRelation.getFrom().equals(asset.getId()) && entityRelation.getTo().equals(device.getId()); + }); + ((Device) deviceExportData.getEntity()).setDeviceProfileId(null); + + Asset importedAsset = importEntity(tenantAdmin2, assetExportData).getSavedEntity(); + Device importedDevice = importEntity(tenantAdmin2, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()).getSavedEntity(); + checkImportedEntity(tenantId1, device, tenantId2, importedDevice); + checkImportedEntity(tenantId1, asset, tenantId2, importedAsset); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); + } + + @Test + public void testExportImportWithRelations_betweenTenants() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation = createRelation(asset.getId(), device.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId()); + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .exportCredentials(false) + .build()); + deviceExportData.getEntity().setDeviceProfileId(null); + + Asset importedAsset = importEntity(tenantAdmin2, assetExportData).getSavedEntity(); + Device importedDevice = importEntity(tenantAdmin2, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()).getSavedEntity(); + + List importedRelations = relationService.findByTo(TenantId.SYS_TENANT_ID, importedDevice.getId(), RelationTypeGroup.COMMON); + assertThat(importedRelations).size().isOne(); + assertThat(importedRelations.get(0)).satisfies(importedRelation -> { + assertThat(importedRelation.getFrom()).isEqualTo(importedAsset.getId()); + assertThat(importedRelation.getType()).isEqualTo(relation.getType()); + assertThat(importedRelation.getAdditionalInfo()).isEqualTo(relation.getAdditionalInfo()); + }); + } + + @Test + public void testExportImportWithRelations_sameTenant() throws Exception { + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + Device device1 = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset.getId(), device1.getId()); + + EntityExportData assetExportData = exportEntity(tenantAdmin1, asset.getId(), EntityExportSettings.builder() + .exportRelations(true) + .build()); + assertThat(assetExportData.getRelations()).size().isOne(); + + Device device2 = createDevice(tenantId1, null, null, "Device 2"); + EntityRelation relation2 = createRelation(asset.getId(), device2.getId()); + + importEntity(tenantAdmin1, assetExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()); + + List relations = relationService.findByFrom(TenantId.SYS_TENANT_ID, asset.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + @Test + public void textExportImportWithRelations_sameTenant_removeExisting() throws Exception { + Asset asset1 = createAsset(tenantId1, null, "A", "Asset 1"); + Device device = createDevice(tenantId1, null, null, "Device 1"); + EntityRelation relation1 = createRelation(asset1.getId(), device.getId()); + + EntityExportData deviceExportData = exportEntity(tenantAdmin1, device.getId(), EntityExportSettings.builder() + .exportRelations(true) + .build()); + assertThat(deviceExportData.getRelations()).size().isOne(); + + Asset asset2 = createAsset(tenantId1, null, "A", "Asset 2"); + EntityRelation relation2 = createRelation(asset2.getId(), device.getId()); + + importEntity(tenantAdmin1, deviceExportData, EntityImportSettings.builder() + .updateRelations(true) + .build()); + + List relations = relationService.findByTo(TenantId.SYS_TENANT_ID, device.getId(), RelationTypeGroup.COMMON); + assertThat(relations).contains(relation1); + assertThat(relations).doesNotContain(relation2); + } + + + @Test + public void testExportImportDeviceProfile_betweenTenants_findExistingByName() throws Exception { + DeviceProfile defaultDeviceProfile = deviceProfileService.findDefaultDeviceProfile(tenantId1); + EntityExportData deviceProfileExportData = exportEntity(tenantAdmin1, defaultDeviceProfile.getId()); + + assertThatThrownBy(() -> { + importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() + .findExistingByName(false) + .build()); + }).hasMessageContaining("default device profile is present"); + + importEntity(tenantAdmin2, deviceProfileExportData, EntityImportSettings.builder() + .findExistingByName(true) + .build()); + checkImportedEntity(tenantId1, defaultDeviceProfile, tenantId2, deviceProfileService.findDefaultDeviceProfile(tenantId2)); + } + + + @SuppressWarnings("rawTypes") + private static EntityExportData getAndClone(Map map, EntityType entityType) { + return JacksonUtil.clone(map.get(entityType)); + } + + @SuppressWarnings({"rawTypes", "unchecked"}) + @Test + public void testEntityEventsOnImport() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, null, "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1"); + Dashboard dashboard = createDashboard(tenantId1, null, "Dashboard 1"); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, null, deviceProfile.getId(), "Device 1"); + + Map entitiesExportData = Stream.of(customer.getId(), asset.getId(), device.getId(), + ruleChain.getId(), dashboard.getId(), deviceProfile.getId()) + .map(entityId -> { + try { + return exportEntity(tenantAdmin1, entityId, EntityExportSettings.builder() + .exportCredentials(false) + .build()); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toMap(EntityExportData::getEntityType, d -> d)); + + Mockito.reset(entityActionService); + Customer importedCustomer = (Customer) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CUSTOMER)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.ADDED), isNull()); + Mockito.reset(entityActionService); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.CUSTOMER)); + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedCustomer.getId()), eq(importedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + + EntityExportData updatedCustomerEntity = getAndClone(entitiesExportData, EntityType.CUSTOMER); + updatedCustomerEntity.getEntity().setEmail("t" + updatedCustomerEntity.getEntity().getEmail()); + Customer updatedCustomer = importEntity(tenantAdmin2, updatedCustomerEntity).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedCustomer.getId()), eq(updatedCustomer), + any(), eq(ActionType.UPDATED), isNull()); + verify(tbClusterService).sendNotificationMsgToEdge(any(), any(), eq(importedCustomer.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + Mockito.reset(entityActionService); + + Asset importedAsset = (Asset) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.ASSET)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.ADDED), isNull()); + importEntity(tenantAdmin2, entitiesExportData.get(EntityType.ASSET)); + verify(entityActionService, Mockito.never()).logEntityAction(any(), eq(importedAsset.getId()), eq(importedAsset), + any(), eq(ActionType.UPDATED), isNull()); + + + EntityExportData updatedAssetEntity = getAndClone(entitiesExportData, EntityType.ASSET); + updatedAssetEntity.getEntity().setLabel("t" + updatedAssetEntity.getEntity().getLabel()); + Asset updatedAsset = importEntity(tenantAdmin2, updatedAssetEntity).getSavedEntity(); + + verify(entityActionService).logEntityAction(any(), eq(importedAsset.getId()), eq(updatedAsset), + any(), eq(ActionType.UPDATED), isNull()); + verify(tbClusterService).sendNotificationMsgToEdge(any(), any(), eq(importedAsset.getId()), any(), any(), eq(EdgeEventActionType.UPDATED)); + + RuleChain importedRuleChain = (RuleChain) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.RULE_CHAIN)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedRuleChain.getId()), eq(importedRuleChain), + any(), eq(ActionType.ADDED), isNull()); + verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedRuleChain.getId()), eq(ComponentLifecycleEvent.CREATED)); + + Dashboard importedDashboard = (Dashboard) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DASHBOARD)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDashboard.getId()), eq(importedDashboard), + any(), eq(ActionType.ADDED), isNull()); + + DeviceProfile importedDeviceProfile = (DeviceProfile) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE_PROFILE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDeviceProfile.getId()), eq(importedDeviceProfile), + any(), eq(ActionType.ADDED), isNull()); + verify(tbClusterService).onDeviceProfileChange(eq(importedDeviceProfile), any()); + verify(tbClusterService).broadcastEntityStateChangeEvent(any(), eq(importedDeviceProfile.getId()), eq(ComponentLifecycleEvent.CREATED)); + verify(tbClusterService).sendNotificationMsgToEdge(any(), any(), eq(importedDeviceProfile.getId()), any(), any(), eq(EdgeEventActionType.ADDED)); + verify(otaPackageStateService).update(eq(importedDeviceProfile), eq(false), eq(false)); + + Device importedDevice = (Device) importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)).getSavedEntity(); + verify(entityActionService).logEntityAction(any(), eq(importedDevice.getId()), eq(importedDevice), + any(), eq(ActionType.ADDED), isNull()); + verify(tbClusterService).onDeviceUpdated(eq(importedDevice), isNull()); + importEntity(tenantAdmin2, getAndClone(entitiesExportData, EntityType.DEVICE)); + verify(tbClusterService, Mockito.never()).onDeviceUpdated(eq(importedDevice), eq(importedDevice)); + + EntityExportData updatedDeviceEntity = getAndClone(entitiesExportData, EntityType.DEVICE); + updatedDeviceEntity.getEntity().setLabel("t" + updatedDeviceEntity.getEntity().getLabel()); + Device updatedDevice = importEntity(tenantAdmin2, updatedDeviceEntity).getSavedEntity(); + verify(tbClusterService).onDeviceUpdated(eq(updatedDevice), eq(importedDevice)); + } + + @Test + public void testExternalIdsInExportData() throws Exception { + Customer customer = createCustomer(tenantId1, "Customer 1"); + Asset asset = createAsset(tenantId1, customer.getId(), "A", "Asset 1"); + RuleChain ruleChain = createRuleChain(tenantId1, "Rule chain 1", asset.getId()); + Dashboard dashboard = createDashboard(tenantId1, customer.getId(), "Dashboard 1", asset.getId()); + DeviceProfile deviceProfile = createDeviceProfile(tenantId1, ruleChain.getId(), dashboard.getId(), "Device profile 1"); + Device device = createDevice(tenantId1, customer.getId(), deviceProfile.getId(), "Device 1"); + EntityView entityView = createEntityView(tenantId1, customer.getId(), device.getId(), "Entity view 1"); + + Map ids = new HashMap<>(); + for (EntityId entityId : List.of(customer.getId(), asset.getId(), ruleChain.getId(), dashboard.getId(), + deviceProfile.getId(), device.getId(), entityView.getId(), ruleChain.getId(), dashboard.getId())) { + EntityExportData exportData = exportEntity(getSecurityUser(tenantAdmin1), entityId); + EntityImportResult importResult = importEntity(getSecurityUser(tenantAdmin2), exportData, EntityImportSettings.builder() + .saveCredentials(false) + .build()); + ids.put(entityId, (EntityId) importResult.getSavedEntity().getId()); + } + + Asset exportedAsset = (Asset) exportEntity(tenantAdmin2, (AssetId) ids.get(asset.getId())).getEntity(); + assertThat(exportedAsset.getCustomerId()).isEqualTo(customer.getId()); + + EntityExportData ruleChainExportData = exportEntity(tenantAdmin2, (RuleChainId) ids.get(ruleChain.getId())); + TbMsgGeneratorNodeConfiguration exportedRuleNodeConfig = ((RuleChainExportData) ruleChainExportData).getMetaData().getNodes().stream() + .filter(node -> node.getType().equals(TbMsgGeneratorNode.class.getName())).findFirst() + .map(RuleNode::getConfiguration).map(config -> JacksonUtil.treeToValue(config, TbMsgGeneratorNodeConfiguration.class)).orElse(null); + assertThat(exportedRuleNodeConfig.getOriginatorId()).isEqualTo(asset.getId().toString()); + + Dashboard exportedDashboard = (Dashboard) exportEntity(tenantAdmin2, (DashboardId) ids.get(dashboard.getId())).getEntity(); + assertThat(exportedDashboard.getAssignedCustomers()).hasOnlyOneElementSatisfying(shortCustomerInfo -> { + assertThat(shortCustomerInfo.getCustomerId()).isEqualTo(customer.getId()); + }); + String exportedEntityAliasAssetId = exportedDashboard.getConfiguration().get("entityAliases").elements().next() + .get("filter").get("entityList").elements().next().asText(); + assertThat(exportedEntityAliasAssetId).isEqualTo(asset.getId().toString()); + + DeviceProfile exportedDeviceProfile = (DeviceProfile) exportEntity(tenantAdmin2, (DeviceProfileId) ids.get(deviceProfile.getId())).getEntity(); + assertThat(exportedDeviceProfile.getDefaultRuleChainId()).isEqualTo(ruleChain.getId()); + assertThat(exportedDeviceProfile.getDefaultDashboardId()).isEqualTo(dashboard.getId()); + + Device exportedDevice = (Device) exportEntity(tenantAdmin2, (DeviceId) ids.get(device.getId())).getEntity(); + assertThat(exportedDevice.getCustomerId()).isEqualTo(customer.getId()); + assertThat(exportedDevice.getDeviceProfileId()).isEqualTo(deviceProfile.getId()); + + EntityView exportedEntityView = (EntityView) exportEntity(tenantAdmin2, (EntityViewId) ids.get(entityView.getId())).getEntity(); + assertThat(exportedEntityView.getCustomerId()).isEqualTo(customer.getId()); + assertThat(exportedEntityView.getEntityId()).isEqualTo(device.getId()); + + deviceProfile.setDefaultDashboardId(null); + deviceProfileService.saveDeviceProfile(deviceProfile); + DeviceProfile importedDeviceProfile = deviceProfileService.findDeviceProfileById(tenantId2, (DeviceProfileId) ids.get(deviceProfile.getId())); + importedDeviceProfile.setDefaultDashboardId(null); + deviceProfileService.saveDeviceProfile(importedDeviceProfile); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index b3ab39fd51..ce43ae3bb5 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -16,8 +16,6 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.junit.After; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.Device; @@ -41,7 +39,6 @@ import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadCon import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.AbstractTransportIntegrationTest; import static org.junit.Assert.assertEquals; @@ -54,12 +51,11 @@ import static org.junit.Assert.assertNotNull; public abstract class AbstractCoapIntegrationTest extends AbstractTransportIntegrationTest { protected final byte[] EMPTY_PAYLOAD = new byte[0]; - - protected CoapClient client; + protected CoapTestClient client; protected void processAfterTest() throws Exception { if (client != null) { - client.shutdown(); + client.disconnect(); } } @@ -156,16 +152,4 @@ public abstract class AbstractCoapIntegrationTest extends AbstractTransportInteg device.setType(type); return doPost("/api/device", device, Device.class); } - - protected CoapClient getCoapClient(FeatureType featureType) { - return new CoapClient(getFeatureTokenUrl(accessToken, featureType)); - } - - protected CoapClient getCoapClient(String featureTokenUrl) { - return new CoapClient(featureTokenUrl); - } - - protected String getFeatureTokenUrl(String token, FeatureType featureType) { - return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); - } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java new file mode 100644 index 0000000000..89fa13220d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.CoapHandler; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; + +import java.util.concurrent.CountDownLatch; + +@Slf4j +@Data +public class CoapTestCallback implements CoapHandler { + + protected final CountDownLatch latch; + protected Integer observe; + protected byte[] payloadBytes; + protected CoAP.ResponseCode responseCode; + + public CoapTestCallback() { + this.latch = new CountDownLatch(1); + } + + public CoapTestCallback(int subscribeCount) { + this.latch = new CountDownLatch(subscribeCount); + } + + public Integer getObserve() { + return observe; + } + + public byte[] getPayloadBytes() { + return payloadBytes; + } + + public CoAP.ResponseCode getResponseCode() { + return responseCode; + } + + @Override + public void onLoad(CoapResponse response) { + observe = response.getOptions().getObserve(); + payloadBytes = response.getPayload(); + responseCode = response.getCode(); + latch.countDown(); + } + + @Override + public void onError() { + log.warn("Command Response Ack Error, No connect"); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java new file mode 100644 index 0000000000..dc2a3f4c83 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapHandler; +import org.eclipse.californium.core.CoapObserveRelation; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; +import org.eclipse.californium.core.coap.MediaTypeRegistry; +import org.eclipse.californium.core.coap.Request; +import org.eclipse.californium.elements.exception.ConnectorException; +import org.thingsboard.server.common.msg.session.FeatureType; + +import java.io.IOException; + +public class CoapTestClient { + + private static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/"; + private static final long CLIENT_REQUEST_TIMEOUT = 60000L; + + private final CoapClient client; + + public CoapTestClient(){ + this.client = createClient(); + } + + public CoapTestClient(String accessToken, FeatureType featureType) { + this.client = createClient(getFeatureTokenUrl(accessToken, featureType)); + } + + public CoapTestClient(String featureTokenUrl) { + this.client = createClient(featureTokenUrl); + } + + public void connectToCoap(String accessToken) { + setURI(accessToken, null); + } + + public void connectToCoap(String accessToken, FeatureType featureType) { + setURI(accessToken, featureType); + } + + public void disconnect() { + if (client != null) { + client.shutdown(); + } + } + + public CoapResponse postMethod(String requestBody) throws ConnectorException, IOException { + return this.postMethod(requestBody.getBytes()); + } + + public CoapResponse postMethod(byte[] requestBodyBytes) throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBodyBytes, MediaTypeRegistry.APPLICATION_JSON); + } + + public void postMethod(CoapHandler handler, String payload, int format) { + client.post(handler, payload, format); + } + + public void postMethod(CoapHandler handler, byte[] payload, int format) { + client.post(handler, payload, format); + } + + public CoapResponse getMethod() throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); + } + + public CoapObserveRelation getObserveRelation(CoapTestCallback callback){ + Request request = Request.newGet().setObserve(); + request.setType(CoAP.Type.CON); + return client.observe(request, callback); + } + + public void setURI(String featureTokenUrl) { + if (client == null) { + throw new RuntimeException("Failed to connect! CoapClient is not initialized!"); + } + client.setURI(featureTokenUrl); + } + + public void setURI(String accessToken, FeatureType featureType) { + if (featureType == null){ + featureType = FeatureType.ATTRIBUTES; + } + setURI(getFeatureTokenUrl(accessToken, featureType)); + } + + private CoapClient createClient() { + return new CoapClient(); + } + + private CoapClient createClient(String featureTokenUrl) { + return new CoapClient(featureTokenUrl); + } + + public static String getFeatureTokenUrl(String token, FeatureType featureType) { + return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); + } + + public static String getFeatureTokenUrl(String token, FeatureType featureType, int requestId) { + return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase() + "/" + requestId; + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index 9d0078aa3f..9710c82866 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -15,25 +15,94 @@ */ package org.thingsboard.server.transport.coap.attributes; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.eclipse.californium.core.CoapObserveRelation; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestCallback; +import org.thingsboard.server.transport.coap.CoapTestClient; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.query.EntityKeyType.CLIENT_ATTRIBUTE; +import static org.thingsboard.server.common.data.query.EntityKeyType.SHARED_ATTRIBUTE; @Slf4j public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { - protected static final String POST_ATTRIBUTES_PAYLOAD = "{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73," + - "\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + @Autowired + DefaultTransportService defaultTransportService; + + public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostAttributes {\n" + + " string clientStr = 1;\n" + + " bool clientBool = 2;\n" + + " double clientDbl = 3;\n" + + " int32 clientLong = 4;\n" + + " JsonObject clientJson = 5;\n" + + "\n" + + " message JsonObject {\n" + + " int32 someNumber = 6;\n" + + " repeated int32 someArray = 7;\n" + + " NestedJsonObject someNestedObject = 8;\n" + + " message NestedJsonObject {\n" + + " string key = 9;\n" + + " }\n" + + " }\n" + + "}"; + + private static final String CLIENT_ATTRIBUTES_PAYLOAD = "{\"clientStr\":\"value1\",\"clientBool\":true,\"clientDbl\":42.0,\"clientLong\":73," + + "\"clientJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + + private static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," + + "\"sharedJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + + protected static final String SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"sharedStr\":\"value\",\"sharedBool\":false,\"sharedDbl\":41.0,\"sharedLong\":72," + + "\"sharedJson\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}"; - protected List getTsKvProtoList() { - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value1", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "true", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "42.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "73", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + private static final String SHARED_ATTRIBUTES_DELETED_RESPONSE = "{\"deleted\":[\"sharedJson\"]}"; + + private List getTsKvProtoList(String attributePrefix) { + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto(attributePrefix + "Str", "value1", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto(attributePrefix + "Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto(attributePrefix + "Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto(attributePrefix + "Long", "73", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto(attributePrefix + "Json", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); List tsKvProtoList = new ArrayList<>(); tsKvProtoList.add(tsKvProtoAttribute1); tsKvProtoList.add(tsKvProtoAttribute2); @@ -49,4 +118,295 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap tsKvProtoBuilder.setKv(keyValueProto); return tsKvProtoBuilder.build(); } + + private List getEntityKeys(List keys, EntityKeyType scope) { + return keys.stream().map(key -> new EntityKey(scope, key)).collect(Collectors.toList()); + } + + private byte[] getAttributesProtoPayloadBytes() { + + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); + DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + + DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); + + DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); + + DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); + Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); + assertNotNull(postAttributesMsgDescriptor); + DynamicMessage postAttributesMsg = postAttributesBuilder + .setField(postAttributesMsgDescriptor.findFieldByName("clientStr"), "value1") + .setField(postAttributesMsgDescriptor.findFieldByName("clientBool"), true) + .setField(postAttributesMsgDescriptor.findFieldByName("clientDbl"), 42.0) + .setField(postAttributesMsgDescriptor.findFieldByName("clientLong"), 73) + .setField(postAttributesMsgDescriptor.findFieldByName("clientJson"), jsonObject) + .build(); + return postAttributesMsg.toByteArray(); + } + + protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exception { + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + SingleEntityFilter dtf = new SingleEntityFilter(); + dtf.setSingleEntity(savedDevice.getId()); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; + List clientKeysList = List.of(clientKeysStr.split(",")); + List sharedKeysList = List.of(sharedKeysStr.split(",")); + List csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE); + List shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE); + List keys = new ArrayList<>(); + keys.addAll(csKeys); + keys.addAll(shKeys); + getWsClient().subscribeLatestUpdate(keys, dtf); + getWsClient().registerWaitForUpdate(2); + + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", + SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + + CoapResponse coapResponse = client.postMethod(CLIENT_ATTRIBUTES_PAYLOAD); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + + String update = getWsClient().waitForUpdate(); + assertThat(update).as("ws update received").isNotBlank(); + + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; + client.setURI(featureTokenUrl); + validateJsonResponse(client.getMethod()); + } + + protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exception { + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + SingleEntityFilter dtf = new SingleEntityFilter(); + dtf.setSingleEntity(savedDevice.getId()); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; + List clientKeysList = List.of(clientKeysStr.split(",")); + List sharedKeysList = List.of(sharedKeysStr.split(",")); + List csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE); + List shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE); + List keys = new ArrayList<>(); + keys.addAll(csKeys); + keys.addAll(shKeys); + getWsClient().subscribeLatestUpdate(keys, dtf); + getWsClient().registerWaitForUpdate(2); + + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", + SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + + CoapResponse coapResponse = client.postMethod(getAttributesProtoPayloadBytes()); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + + String update = getWsClient().waitForUpdate(); + assertThat(update).as("ws update received").isNotBlank(); + + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; + client.setURI(featureTokenUrl); + validateProtoResponse(client.getMethod()); + } + + protected void processJsonTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { + if (!emptyCurrentStateNotification) { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); + } + + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapTestCallback callbackCoap = new CoapTestCallback(1); + + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + + if (emptyCurrentStateNotification) { + validateUpdateAttributesJsonResponse(callbackCoap, "{}", 0); + } else { + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, 0); + } + + CountDownLatch latch = new CountDownLatch(1); + int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1; + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + latch.await(3, TimeUnit.SECONDS); + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD, expectedObserveCnt); + + latch = new CountDownLatch(1); + int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1; + doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); + latch.await(3, TimeUnit.SECONDS); + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_DELETED_RESPONSE, expectedObserveBeforeDeleteCnt); + + observeRelation.proactiveCancel(); + assertTrue(observeRelation.isCanceled()); + + awaitClientAfterCancelObserve(); + } + + protected void processProtoTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { + if (!emptyCurrentStateNotification) { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); + } + + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapTestCallback callbackCoap = new CoapTestCallback(1); + + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + + if (emptyCurrentStateNotification) { + validateEmptyCurrentStateAttributesProtoResponse(callbackCoap); + } else { + validateCurrentStateAttributesProtoResponse(callbackCoap); + } + + CountDownLatch latch = new CountDownLatch(1); + int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1; + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + latch.await(3, TimeUnit.SECONDS); + validateUpdateProtoAttributesResponse(callbackCoap, expectedObserveCnt); + + latch = new CountDownLatch(1); + int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1; + doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); + latch.await(3, TimeUnit.SECONDS); + validateDeleteProtoAttributesResponse(callbackCoap, expectedObserveBeforeDeleteCnt); + + observeRelation.proactiveCancel(); + assertTrue(observeRelation.isCanceled()); + + awaitClientAfterCancelObserve(); + } + + protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { + assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); + String expectedResponse = "{\"client\":" + CLIENT_ATTRIBUTES_PAYLOAD + ",\"shared\":" + SHARED_ATTRIBUTES_PAYLOAD + "}"; + assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(getAttributesResponse.getPayload())); + } + + protected void validateProtoResponse(CoapResponse getAttributesResponse) throws InterruptedException, InvalidProtocolBufferException { + TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); + TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); + assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); + List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); + assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); + } + + protected void validateUpdateAttributesJsonResponse(CoapTestCallback callback, String expectedResponse, int expectedObserveCnt) { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); + assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.toJsonNode(response)); + } + + protected void validateEmptyCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException { + assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(0, callback.getObserve().intValue()); + } + + protected void validateCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(0, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("sharedStr", "value", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("sharedBool", "false", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("sharedDbl", "41.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("sharedLong", "72", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("sharedJson", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + List tsKvProtoList = new ArrayList<>(); + tsKvProtoList.add(tsKvProtoAttribute1); + tsKvProtoList.add(tsKvProtoAttribute2); + tsKvProtoList.add(tsKvProtoAttribute3); + tsKvProtoList.add(tsKvProtoAttribute4); + tsKvProtoList.add(tsKvProtoAttribute5); + TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build(); + TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + List expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + + assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); + assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); + } + + protected void validateUpdateProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + List tsKvProtoList = getTsKvProtoList("shared"); + attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); + + TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); + TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + List actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + + assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); + assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); + } + + protected void validateDeleteProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); + + TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); + TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size()); + assertEquals("sharedJson", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0)); + } + + private void awaitClientAfterCancelObserve() { + Awaitility.await("awaitClientAfterCancelObserve") + .pollInterval(10, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .until(()->{ + log.trace("awaiting defaultTransportService.sessions is empty"); + return defaultTransportService.sessions.isEmpty();}); + } + + private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { + TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); + List csTsKvProtoList = getTsKvProtoList("client"); + List shTsKvProtoList = getTsKvProtoList("shared"); + result.addAllClientAttributeList(csTsKvProtoList); + result.addAllSharedAttributeList(shTsKvProtoList); + result.setRequestId(0); + return result.build(); + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java index 977502e0e6..727fb52380 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java @@ -15,34 +15,18 @@ */ package org.thingsboard.server.transport.coap.attributes.request; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; -import org.thingsboard.server.common.msg.session.FeatureType; - -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j @DaoSqlTest public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributesIntegrationTest { - protected static final long CLIENT_REQUEST_TIMEOUT = 60000L; - @Before public void beforeTest() throws Exception { CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() @@ -58,44 +42,6 @@ public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributes @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); - } - - protected void processTestRequestAttributesValuesFromTheServer() throws Exception { - postAttributes(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; - - List savedAttributeKeys = null; - while (start <= end) { - savedAttributeKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (savedAttributeKeys.size() == 5) { - break; - } - Thread.sleep(100); - start += 100; - } - assertNotNull(savedAttributeKeys); - - String keys = "attribute1,attribute2,attribute3,attribute4,attribute5"; - String featureTokenUrl = getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; - client = getCoapClient(featureTokenUrl); - - CoapResponse getAttributesResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); - validateResponse(getAttributesResponse); - } - - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - client = getCoapClient(FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { - assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); - String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; - assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(getAttributesResponse.getPayload(), StandardCharsets.UTF_8))); + processJsonTestRequestAttributesValuesFromTheServer(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java index e421fbf1a0..2c6928b8d4 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java @@ -45,6 +45,6 @@ public class CoapAttributesRequestJsonIntegrationTest extends CoapAttributesRequ @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - super.testRequestAttributesValuesFromTheServer(); + processJsonTestRequestAttributesValuesFromTheServer(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java index 7bdcce1400..5f300ba68e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java @@ -15,64 +15,18 @@ */ package org.thingsboard.server.transport.coap.attributes.request; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @Slf4j @DaoSqlTest public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesRequestIntegrationTest { - public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + - "\n" + - "package test;\n" + - "\n" + - "message PostAttributes {\n" + - " string attribute1 = 1;\n" + - " bool attribute2 = 2;\n" + - " double attribute3 = 3;\n" + - " int32 attribute4 = 4;\n" + - " JsonObject attribute5 = 5;\n" + - "\n" + - " message JsonObject {\n" + - " int32 someNumber = 6;\n" + - " repeated int32 someArray = 7;\n" + - " NestedJsonObject someNestedObject = 8;\n" + - " message NestedJsonObject {\n" + - " string key = 9;\n" + - " }\n" + - " }\n" + - "}"; - @Before @Override public void beforeTest() throws Exception { @@ -87,74 +41,6 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); + processProtoTestRequestAttributesValuesFromTheServer(); } - - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - - DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); - Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); - assertNotNull(nestedJsonObjectBuilderDescriptor); - DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); - - DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); - Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); - assertNotNull(jsonObjectBuilderDescriptor); - DynamicMessage jsonObject = jsonObjectBuilder - .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) - .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) - .build(); - - DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); - Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); - assertNotNull(postAttributesMsgDescriptor); - DynamicMessage postAttributesMsg = postAttributesBuilder - .setField(postAttributesMsgDescriptor.findFieldByName("attribute1"), "value1") - .setField(postAttributesMsgDescriptor.findFieldByName("attribute2"), true) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute3"), 42.0) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute4"), 73) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute5"), jsonObject) - .build(); - byte[] payload = postAttributesMsg.toByteArray(); - client = getCoapClient(FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { - TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); - TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); - assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); - List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); - assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); - } - - private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { - TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); - List tsKvProtoList = getTsKvProtoList(); - result.addAllClientAttributeList(tsKvProtoList); - result.addAllSharedAttributeList(tsKvProtoList); - result.setRequestId(0); - return result.build(); - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java index bbbfad3c65..3214ae1639 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java @@ -15,47 +15,25 @@ */ package org.thingsboard.server.transport.coap.attributes.updates; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.awaitility.Awaitility; -import org.eclipse.californium.core.CoapHandler; -import org.eclipse.californium.core.CoapObserveRelation; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.server.resources.Resource; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.coapserver.DefaultCoapServerService; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import org.thingsboard.server.transport.coap.CoapTransportResource; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.spy; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j @DaoSqlTest public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributesIntegrationTest { - private static final String RESPONSE_ATTRIBUTES_PAYLOAD_DELETED = "{\"deleted\":[\"attribute5\"]}"; - - protected static final String POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"attribute1\":\"value\",\"attribute2\":false,\"attribute3\":41.0,\"attribute4\":72," + - "\"attribute5\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}"; - CoapTransportResource coapTransportResource; @Autowired @@ -83,136 +61,11 @@ public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributes @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - processTestSubscribeToAttributesUpdates(false); + processJsonTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - processTestSubscribeToAttributesUpdates(true); - } - - protected void processTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { - if (!emptyCurrentStateNotification) { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); - } - client = getCoapClient(FeatureType.ATTRIBUTES); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(latch); - - Request request = Request.newGet().setObserve(); - request.setType(CoAP.Type.CON); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); - - if (emptyCurrentStateNotification) { - validateEmptyCurrentStateAttributesResponse(callback); - } else { - validateCurrentStateAttributesResponse(callback); - } - - latch = new CountDownLatch(1); - int expectedObserveCnt = callback.getObserve().intValue() + 1; - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateUpdateAttributesResponse(callback, expectedObserveCnt); - - - latch = new CountDownLatch(1); - int expectedObserveBeforeDeleteCnt = callback.getObserve().intValue() + 1; - doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=attribute5", String.class); - latch.await(3, TimeUnit.SECONDS); - - validateDeleteAttributesResponse(callback, expectedObserveBeforeDeleteCnt); - observeRelation.proactiveCancel(); - assertTrue(observeRelation.isCanceled()); - - awaitClientAfterCancelObserve(); - } - - protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION), JacksonUtil.toJsonNode(response)); - } - - protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals("{}", response); - } - - protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD), JacksonUtil.toJsonNode(response)); - } - - protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(RESPONSE_ATTRIBUTES_PAYLOAD_DELETED), JacksonUtil.toJsonNode(response)); - } - - protected static class TestCoapCallback implements CoapHandler { - - private final CountDownLatch latch; - - private Integer observe; - private byte[] payloadBytes; - private CoAP.ResponseCode responseCode; - - public Integer getObserve() { - return observe; - } - - public byte[] getPayloadBytes() { - return payloadBytes; - } - - public CoAP.ResponseCode getResponseCode() { - return responseCode; - } - - private TestCoapCallback(CountDownLatch latch) { - this.latch = latch; - } - - @Override - public void onLoad(CoapResponse response) { - observe = response.getOptions().getObserve(); - payloadBytes = response.getPayload(); - responseCode = response.getCode(); - latch.countDown(); - } - - @Override - public void onError() { - log.warn("Command Response Ack Error, No connect"); - } - - } - - private void awaitClientAfterCancelObserve() { - Awaitility.await("awaitClientAfterCancelObserve") - .pollInterval(10, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .until(()->{ - log.trace("awaiting defaultTransportService.sessions is empty"); - return defaultTransportService.sessions.isEmpty();}); + processJsonTestSubscribeToAttributesUpdates(true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java index 4ab052db46..816ffa53b3 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java @@ -23,10 +23,11 @@ import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; +import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; @Slf4j @DaoSqlTest -public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpdatesIntegrationTest { +public class CoapAttributesUpdatesJsonIntegrationTest extends AbstractCoapAttributesIntegrationTest { @Before public void beforeTest() throws Exception { @@ -45,11 +46,11 @@ public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpda @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - super.testSubscribeToAttributesUpdatesFromTheServer(); + processJsonTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - super.testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification(); + processJsonTestSubscribeToAttributesUpdates(true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java index 4d9970b2dc..4a5e7c033f 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java @@ -15,30 +15,19 @@ */ package org.thingsboard.server.transport.coap.attributes.updates; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.coap.CoAP; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertTrue; +import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; @Slf4j @DaoSqlTest -public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpdatesIntegrationTest { +public class CoapAttributesUpdatesProtoIntegrationTest extends AbstractCoapAttributesIntegrationTest { @Before public void beforeTest() throws Exception { @@ -57,83 +46,11 @@ public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpd @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - processTestSubscribeToAttributesUpdates(false); + processProtoTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - processTestSubscribeToAttributesUpdates(true); + processProtoTestSubscribeToAttributesUpdates(true); } - - protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "false", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "41.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "72", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); - List tsKvProtoList = new ArrayList<>(); - tsKvProtoList.add(tsKvProtoAttribute1); - tsKvProtoList.add(tsKvProtoAttribute2); - tsKvProtoList.add(tsKvProtoAttribute3); - tsKvProtoList.add(tsKvProtoAttribute4); - tsKvProtoList.add(tsKvProtoAttribute5); - TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build(); - TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - List expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - - assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); - assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); - - } - - protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - } - - protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - List tsKvProtoList = getTsKvProtoList(); - attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); - - TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); - TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - List actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - - assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); - assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); - - } - - protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - attributeUpdateNotificationMsgBuilder.addSharedDeleted("attribute5"); - - TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); - TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size()); - assertEquals("attribute5", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0)); - - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java index de9c024bb8..e2be2280b2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java @@ -16,10 +16,8 @@ package org.thingsboard.server.transport.coap.claim; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; @@ -34,6 +32,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import java.io.IOException; @@ -96,7 +95,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { log.warn("[testClaimingDevice] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType()); - client = getCoapClient(FeatureType.CLAIM); + client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; byte[] failurePayloadBytes; if (emptyPayload) { @@ -109,7 +108,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { validateClaimResponse(emptyPayload, client, payloadBytes, failurePayloadBytes); } - protected void validateClaimResponse(boolean emptyPayload, CoapClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception { + protected void validateClaimResponse(boolean emptyPayload, CoapTestClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception { postClaimRequest(client, failurePayloadBytes); loginUser(customerAdmin.getName(), CUSTOMER_USER_PASSWORD); @@ -145,8 +144,8 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { assertEquals(claimResponse, ClaimResponse.CLAIMED); } - private void postClaimRequest(CoapClient client, byte[] payload) throws IOException, ConnectorException { - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + private void postClaimRequest(CoapTestClient client, byte[] payload) throws IOException, ConnectorException { + CoapResponse coapResponse = client.postMethod(payload); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java index 812ea7392b..0096827af7 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportApiProtos; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; @Slf4j @@ -56,7 +57,7 @@ public class CoapClaimProtoDeviceTest extends CoapClaimDeviceTest { @Override protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { - client = getCoapClient(FeatureType.CLAIM); + client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; if (emptyPayload) { TransportApiProtos.ClaimDevice claimDevice = getClaimDevice(0, emptyPayload); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java index a952ef6d63..ce5ca909ba 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java @@ -15,16 +15,13 @@ */ package org.thingsboard.server.transport.coap.provision; -import com.google.gson.JsonObject; +import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.common.data.CoapDeviceType; @@ -34,14 +31,12 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.session.FeatureType; -import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; - @Slf4j @DaoSqlTest public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { @@ -95,10 +90,11 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .transportPayloadType(TransportPayloadType.JSON) .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); - Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("errorMsg")); + Assert.assertTrue(response.hasNonNull("status")); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText()); } @@ -112,8 +108,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -121,8 +118,8 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } @@ -137,8 +134,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .build(); processBeforeTest(configProperties); String requestCredentials = ",\"credentialsType\": \"ACCESS_TOKEN\",\"token\": \"test_token\""; - byte[] result = createCoapClientAndPublish(requestCredentials).getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials)); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -146,10 +144,10 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "ACCESS_TOKEN"); Assert.assertEquals(deviceCredentials.getCredentialsId(), "test_token"); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } @@ -164,8 +162,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .build(); processBeforeTest(configProperties); String requestCredentials = ",\"credentialsType\": \"X509_CERTIFICATE\",\"hash\": \"testHash\""; - byte[] result = createCoapClientAndPublish(requestCredentials).getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials)); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -173,7 +172,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "X509_CERTIFICATE"); String cert = EncryptionUtil.certTrimNewLines(deviceCredentials.getCredentialsValue()); @@ -182,7 +181,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { Assert.assertEquals(deviceCredentials.getCredentialsId(), sha3Hash); Assert.assertEquals(deviceCredentials.getCredentialsValue(), "testHash"); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } private void processTestProvisioningCheckPreProvisionedDevice() throws Exception { @@ -195,13 +194,14 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } private void processTestProvisioningWithBadKeyDevice() throws Exception { @@ -214,25 +214,21 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); - Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("errorMsg")); + Assert.assertTrue(response.hasNonNull("status")); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText()); } - private CoapResponse createCoapClientAndPublish() throws Exception { + private byte[] createCoapClientAndPublish() throws Exception { return createCoapClientAndPublish(""); } - private CoapResponse createCoapClientAndPublish(String deviceCredentials) throws Exception { + private byte[] createCoapClientAndPublish(String deviceCredentials) throws Exception { String provisionRequestMsg = createTestProvisionMessage(deviceCredentials); - client = getCoapClient(FeatureType.PROVISION); - return postProvision(client, provisionRequestMsg.getBytes()); - } - - - private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException { - return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + client = new CoapTestClient(accessToken, FeatureType.PROVISION); + return client.postMethod(provisionRequestMsg.getBytes()).getPayload(); } private String createTestProvisionMessage(String deviceCredentials) { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java index a2e270796e..a5253b8409 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java @@ -16,10 +16,7 @@ package org.thingsboard.server.transport.coap.provision; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -44,10 +41,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceReque import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; - @Slf4j @DaoSqlTest public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { @@ -101,9 +97,9 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .transportPayloadType(TransportPayloadType.PROTOBUF) .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); Assert.assertNotNull(result); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().toString()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().name()); } private void processTestProvisioningCreateNewDeviceWithoutCredentials() throws Exception { @@ -116,7 +112,7 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -124,8 +120,8 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name()); } private void processTestProvisioningCreateNewDeviceWithAccessToken() throws Exception { @@ -138,9 +134,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()).build(); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder() + .setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()) + .build(); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials)).getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom( + createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials))); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -164,9 +163,13 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceX509CertRequestMsg(ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()).build(); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder() + .setValidateDeviceX509CertRequestMsg( + ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()) + .build(); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials)).getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom( + createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials))); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -196,12 +199,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name()); } private void processTestProvisioningWithBadKeyDevice() throws Exception { @@ -214,24 +217,19 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().toString()); - } - - private CoapResponse createCoapClientAndPublish() throws Exception { - byte[] provisionRequestMsg = createTestProvisionMessage(); - CoapResponse coapResponse = createCoapClientAndPublish(provisionRequestMsg); - Assert.assertNotNull("COAP response", coapResponse); - return coapResponse; + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().name()); } - private CoapResponse createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { - client = getCoapClient(FeatureType.PROVISION); - return postProvision(client, provisionRequestMsg); + private byte[] createCoapClientAndPublish() throws Exception { + return createCoapClientAndPublish(createTestProvisionMessage()); } - private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException { - return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + private byte[] createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { + client = new CoapTestClient(accessToken, FeatureType.PROVISION); + CoapResponse coapResponse = client.postMethod(provisionRequestMsg); + Assert.assertNotNull("COAP response", coapResponse); + return coapResponse.getPayload(); } private byte[] createTestsProvisionMessage(CredentialsType credentialsType, CredentialsDataProto credentialsData) throws Exception { @@ -247,7 +245,6 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .toByteArray(); } - private byte[] createTestProvisionMessage() throws Exception { return createTestsProvisionMessage(null, null); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java index 5262b55b97..beb4d3747c 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java @@ -16,18 +16,29 @@ package org.thingsboard.server.transport.coap.rpc; import com.fasterxml.jackson.databind.JsonNode; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapHandler; import org.eclipse.californium.core.CoapObserveRelation; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.core.coap.Request; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestCallback; +import org.thingsboard.server.transport.coap.CoapTestClient; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -41,76 +52,74 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractCoapIntegrationTest { + public static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" + + "package rpc;\n" + + "\n" + + "message RpcRequestMsg {\n" + + " optional string method = 1;\n" + + " optional int32 requestId = 2;\n" + + " Params params = 3;\n" + + "\n" + + " message Params {\n" + + " optional string pin = 1;\n" + + " optional int32 value = 2;\n" + + " }\n" + + "}"; + protected static final String DEVICE_RESPONSE = "{\"value1\":\"A\",\"value2\":\"B\"}"; protected static final Long asyncContextTimeoutToUseRpcPlugin = 10000L; - protected void processOneWayRpcTest() throws Exception { - client = getCoapClient(FeatureType.RPC); - client.useCONs(); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(client, latch, true); + protected void processOneWayRpcTest(boolean protobuf) throws Exception { + client = new CoapTestClient(accessToken, FeatureType.RPC); + CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, true, protobuf); - Request request = Request.newGet().setObserve(); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); - - validateCurrentStateNotification(callback); - - latch = new CountDownLatch(1); + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + validateCurrentStateNotification(callbackCoap); + CountDownLatch latch = new CountDownLatch(1); String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateOneWayStateChangedNotification(callback, result); + validateOneWayStateChangedNotification(callbackCoap, result); observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); } - protected void processTwoWayRpcTest(String expectedResponseResult) throws Exception { - client = getCoapClient(FeatureType.RPC); - client.useCONs(); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(client, latch, false); + protected void processTwoWayRpcTest(String expectedResponseResult, boolean protobuf) throws Exception { + client = new CoapTestClient(accessToken, FeatureType.RPC); + CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, false, protobuf); - Request request = Request.newGet().setObserve(); - request.setType(CoAP.Type.CON); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - validateCurrentStateNotification(callback); + validateCurrentStateNotification(callbackCoap); String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateTwoWayStateChangedNotification(callback, 1, expectedResponseResult, actualResult); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - latch = new CountDownLatch(1); + validateTwoWayStateChangedNotification(callbackCoap, 1, expectedResponseResult, actualResult); + CountDownLatch latch = new CountDownLatch(1); actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - validateTwoWayStateChangedNotification(callback, 2, expectedResponseResult, actualResult); + validateTwoWayStateChangedNotification(callbackCoap, 2, expectedResponseResult, actualResult); observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); } - protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { + protected void processOnLoadResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { JsonNode responseJson = JacksonUtil.fromBytes(response.getPayload()); - client.setURI(getRpcResponseFeatureTokenUrl(accessToken, responseJson.get("id").asInt())); - client.post(new CoapHandler() { + client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, responseJson.get("id").asInt())); + client.postMethod(new CoapHandler() { @Override public void onLoad(CoapResponse response) { log.warn("Command Response Ack: {}, {}", response.getCode(), response.getResponseText()); @@ -124,36 +133,89 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC }, DEVICE_RESPONSE, MediaTypeRegistry.APPLICATION_JSON); } - protected String getRpcResponseFeatureTokenUrl(String token, int requestId) { - return COAP_BASE_URL + token + "/" + FeatureType.RPC.name().toLowerCase() + "/" + requestId; + protected void processOnLoadProtoResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); + ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); + DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + + byte[] requestPayload = response.getPayload(); + DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); + Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType(); + try { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); + Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); + int requestId = (int) dynamicMessage.getField(requestIdDescriptor); + ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); + DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); + Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); + DynamicMessage rpcResponseMsg = rpcResponseBuilder + .setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE) + .build(); + client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, requestId)); + client.postMethod(new CoapHandler() { + @Override + public void onLoad(CoapResponse response) { + log.warn("Command Response Ack: {}", response.getCode()); + latch.countDown(); + } + + @Override + public void onError() { + log.warn("Command Response Ack Error, No connect"); + } + }, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON); + } catch (InvalidProtocolBufferException e) { + log.warn("Command Response Ack Error, Invalid response received: ", e); + } } - protected class TestCoapCallback implements CoapHandler { + private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() { + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + } - private final CoapClient client; - private final CountDownLatch latch; - private final boolean isOneWayRpc; + private void validateCurrentStateNotification(CoapTestCallback callback) { + assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID); + assertEquals(0, callback.getObserve().intValue()); + } - private Integer observe; - private byte[] payloadBytes; - private CoAP.ResponseCode responseCode; + private void validateOneWayStateChangedNotification(CoapTestCallback callback, String result) { + assertTrue(StringUtils.isEmpty(result)); + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(1, callback.getObserve().intValue()); + } - public Integer getObserve() { - return observe; - } + private void validateTwoWayStateChangedNotification(CoapTestCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) { + assertEquals(expectedResult, actualResult); + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveNumber, callback.getObserve().intValue()); + } - public byte[] getPayloadBytes() { - return payloadBytes; - } + protected class TestCoapCallbackForRPC extends CoapTestCallback { - public CoAP.ResponseCode getResponseCode() { - return responseCode; - } + private final CoapTestClient client; + private final boolean isOneWayRpc; + private final boolean protobuf; - TestCoapCallback(CoapClient client, CountDownLatch latch, boolean isOneWayRpc) { + TestCoapCallbackForRPC(CoapTestClient client, int subscribeCount, boolean isOneWayRpc, boolean protobuf) { + super(subscribeCount); this.client = client; - this.latch = latch; this.isOneWayRpc = isOneWayRpc; + this.protobuf = protobuf; } @Override @@ -163,7 +225,11 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC observe = response.getOptions().getObserve(); if (observe != null) { if (!isOneWayRpc && observe > 0) { - processOnLoadResponse(response, client, observe, latch); + if (!protobuf){ + processOnLoadResponse(response, client, observe, latch); + } else { + processOnLoadProtoResponse(response, client, observe, latch); + } } else { latch.countDown(); } @@ -174,31 +240,5 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC public void onError() { log.warn("Command Response Ack Error, No connect"); } - - } - - private void validateCurrentStateNotification(TestCoapCallback callback) { - assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID); - assertEquals(0, callback.getObserve().intValue()); } - - private void validateOneWayStateChangedNotification(TestCoapCallback callback, String result) { - assertTrue(StringUtils.isEmpty(result)); - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(1, callback.getObserve().intValue()); - } - - private void validateTwoWayStateChangedNotification(TestCoapCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) { - assertEquals(expectedResult, actualResult); - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveNumber, callback.getObserve().intValue()); - } - - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java index cd0e9268b4..d2ef37f795 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java @@ -21,9 +21,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; -import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; @@ -87,12 +84,12 @@ public class CoapServerSideRpcDefaultIntegrationTest extends AbstractCoapServerS @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(false); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}"); + processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java index 6dde4b919b..96ab02325a 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java @@ -45,12 +45,12 @@ public class CoapServerSideRpcJsonIntegrationTest extends AbstractCoapServerSide @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(false); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}"); + processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java index ae1c89d765..93b5c6c6fb 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java @@ -15,52 +15,19 @@ */ package org.thingsboard.server.transport.coap.rpc; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.eclipse.californium.core.CoapHandler; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.util.concurrent.CountDownLatch; - -import static org.junit.Assert.assertTrue; - @Slf4j @DaoSqlTest public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSideRpcIntegrationTest { - private static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" + - "package rpc;\n" + - "\n" + - "message RpcRequestMsg {\n" + - " optional string method = 1;\n" + - " optional int32 requestId = 2;\n" + - " Params params = 3;\n" + - "\n" + - " message Params {\n" + - " optional string pin = 1;\n" + - " optional int32 value = 2;\n" + - " }\n" + - "}"; - @Before public void beforeTest() throws Exception { CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() @@ -79,61 +46,11 @@ public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSid @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(true); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}"); - } - - @Override - protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); - - byte[] requestPayload = response.getPayload(); - DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); - Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType(); - try { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); - Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); - int requestId = (int) dynamicMessage.getField(requestIdDescriptor); - ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); - DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); - Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); - DynamicMessage rpcResponseMsg = rpcResponseBuilder - .setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE) - .build(); - client.setURI(getRpcResponseFeatureTokenUrl(accessToken, requestId)); - client.post(new CoapHandler() { - @Override - public void onLoad(CoapResponse response) { - log.warn("Command Response Ack: {}", response.getCode()); - latch.countDown(); - } - - @Override - public void onError() { - log.warn("Command Response Ack Error, No connect"); - } - }, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON); - } catch (InvalidProtocolBufferException e) { - log.warn("Command Response Ack Error, Invalid response received: ", e); - } - } - - private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}", true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java index ee566be2bb..057e4366de 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java @@ -17,11 +17,8 @@ package org.thingsboard.server.transport.coap.telemetry.attributes; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -29,9 +26,9 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashMap; @@ -74,30 +71,17 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { } protected void processAttributesTest(List expectedKeys, byte[] payload, boolean presenceFieldsTest) throws Exception { - client = getCoapClient(FeatureType.ATTRIBUTES); - postAttributes(client, payload); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapResponse coapResponse = client.postMethod(payload); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); DeviceId deviceId = savedDevice.getId(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; - - List actualKeys = null; - while (start <= end) { - actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (actualKeys.size() == expectedKeys.size()) { - break; - } - Thread.sleep(100); - start += 100; - } + List actualKeys = getActualKeysList(deviceId, expectedKeys); assertNotNull(actualKeys); Set actualKeySet = new HashSet<>(actualKeys); - Set expectedKeySet = new HashSet<>(expectedKeys); - assertEquals(expectedKeySet, actualKeySet); String getAttributesValuesUrl = getAttributesValuesUrl(deviceId, actualKeySet); @@ -111,14 +95,6 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { doDelete(deleteAttributesUrl); } - private void postAttributes(CoapClient client, byte[] payload) throws IOException, ConnectorException { - if (payload == null) { - payload = PAYLOAD_VALUES_STR.getBytes(); - } - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - @SuppressWarnings({"unchecked", "rawtypes"}) protected void assertAttributesValues(List> deviceValues, Set keySet) { for (Map map : deviceValues) { @@ -171,6 +147,22 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { } } + private List getActualKeysList(DeviceId deviceId, List expectedKeys) throws Exception { + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List actualKeys = null; + while (start <= end) { + actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); + if (actualKeys.size() == expectedKeys.size()) { + break; + } + Thread.sleep(100); + start += 100; + } + return actualKeys; + } + private String getAttributesValuesUrl(DeviceId deviceId, Set actualKeySet) { return "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/attributes/CLIENT_SCOPE?keys=" + String.join(",", actualKeySet); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java index 0c7ab4d1c5..a1fba145a9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java @@ -17,19 +17,17 @@ package org.thingsboard.server.transport.coap.telemetry.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -75,28 +73,16 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } protected void processTestPostTelemetry(byte[] payloadBytes, List expectedKeys, boolean withTs, boolean presenceFieldsTest) throws Exception { - client = getCoapClient(FeatureType.TELEMETRY); - postTelemetry(client, payloadBytes); - - String deviceId = savedDevice.getId().getId().toString(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; + client = new CoapTestClient(accessToken, FeatureType.TELEMETRY); + CoapResponse coapResponse = client.postMethod(payloadBytes); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - List actualKeys = null; - while (start <= end) { - actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {}); - if (actualKeys.size() == expectedKeys.size()) { - break; - } - Thread.sleep(100); - start += 100; - } + DeviceId deviceId = savedDevice.getId(); + List actualKeys = getActualKeysList(deviceId, expectedKeys); assertNotNull(actualKeys); Set actualKeySet = new HashSet<>(actualKeys); Set expectedKeySet = new HashSet<>(expectedKeys); - assertEquals(expectedKeySet, actualKeySet); String getTelemetryValuesUrl; @@ -105,8 +91,8 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } else { getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); } - start = System.currentTimeMillis(); - end = System.currentTimeMillis() + 5000; + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; Map>> values = null; while (start <= end) { values = doGetAsyncTyped(getTelemetryValuesUrl, new TypeReference<>() {}); @@ -144,11 +130,6 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } } - private void postTelemetry(CoapClient client, byte[] payload) throws IOException, ConnectorException { - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - private void assertTs(Map>> deviceValues, List expectedKeys, int ts, int arrayIndex) { assertEquals(ts, deviceValues.get(expectedKeys.get(0)).get(arrayIndex).get("ts")); assertEquals(ts, deviceValues.get(expectedKeys.get(1)).get(arrayIndex).get("ts")); @@ -207,4 +188,20 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } } + private List getActualKeysList(DeviceId deviceId, List expectedKeys) throws Exception { + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List actualKeys = null; + while (start <= end) { + actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {}); + if (actualKeys.size() == expectedKeys.size()) { + break; + } + Thread.sleep(100); + start += 100; + } + return actualKeys; + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java index 93e132663e..fb6d8aa0da 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 518c9b42d3..279d1e99be 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -55,3 +55,5 @@ queue.rule-engine.queues[2].partitions=2 queue.rule-engine.queues[2].processing-strategy.retries=1 queue.rule-engine.queues[2].processing-strategy.pause-between-retries=0 queue.rule-engine.queues[2].processing-strategy.max-pause-between-retries=0 + +usage.stats.report.enabled=false \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index ed39bf3716..37207798fb 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -21,25 +21,33 @@ import org.springframework.cache.support.NullValue; import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.data.redis.connection.RedisStringCommands; +import org.springframework.data.redis.connection.jedis.JedisClusterConnection; +import org.springframework.data.redis.connection.jedis.JedisConnection; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import org.springframework.data.redis.core.types.Expiration; import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.util.JedisClusterCRC16; import java.io.Serializable; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; @Slf4j public abstract class RedisTbTransactionalCache implements TbTransactionalCache { private static final byte[] BINARY_NULL_VALUE = RedisSerializer.java().serialize(NullValue.INSTANCE); + static final JedisPool MOCK_POOL = new JedisPool(); //non-null pool required for JedisConnection to trigger closing jedis connection @Getter private final String cacheName; - private final RedisConnectionFactory connectionFactory; - private final RedisSerializer keySerializer = new StringRedisSerializer(); + private final JedisConnectionFactory connectionFactory; + private final RedisSerializer keySerializer = StringRedisSerializer.UTF_8; private final RedisSerializer valueSerializer; private final Expiration evictExpiration; private final Expiration cacheTtl; @@ -50,15 +58,15 @@ public abstract class RedisTbTransactionalCache valueSerializer) { this.cacheName = cacheName; - this.connectionFactory = connectionFactory; + this.connectionFactory = (JedisConnectionFactory) connectionFactory; this.valueSerializer = valueSerializer; this.evictExpiration = Expiration.from(configuration.getEvictTtlInMs(), TimeUnit.MILLISECONDS); - if (cacheSpecsMap.getSpecs() != null && cacheSpecsMap.getSpecs().get(cacheName) != null) { - CacheSpecs cacheSpecs = cacheSpecsMap.getSpecs().get(cacheName); - this.cacheTtl = Expiration.from(cacheSpecs.getTimeToLiveInMinutes(), TimeUnit.MINUTES); - } else { - this.cacheTtl = Expiration.persistent(); - } + this.cacheTtl = Optional.ofNullable(cacheSpecsMap) + .map(CacheSpecsMap::getSpecs) + .map(x -> x.get(cacheName)) + .map(CacheSpecs::getTimeToLiveInMinutes) + .map(t -> Expiration.from(t, TimeUnit.MINUTES)) + .orElseGet(Expiration::persistent); } @Override @@ -130,8 +138,23 @@ public abstract class RedisTbTransactionalCache(this, connection); } + private RedisConnection getConnection(byte[] rawKey) { + if (!connectionFactory.isRedisClusterAware()) { + return connectionFactory.getConnection(); + } + RedisConnection connection = connectionFactory.getClusterConnection(); + + int slotNum = JedisClusterCRC16.getSlot(rawKey); + Jedis jedis = ((JedisClusterConnection) connection).getNativeConnection().getConnectionFromSlot(slotNum); + + JedisConnection jedisConnection = new JedisConnection(jedis, MOCK_POOL, jedis.getDB()); + jedisConnection.setConvertPipelineAndTxResults(connectionFactory.getConvertPipelineAndTxResults()); + + return jedisConnection; + } + private RedisConnection watch(byte[][] rawKeysList) { - var connection = connectionFactory.getConnection(); + RedisConnection connection = getConnection(rawKeysList[0]); try { connection.watch(rawKeysList); connection.multi(); diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java index 664eaa1fb4..9cb139e26a 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbCaffeineCacheConfiguration.java @@ -67,7 +67,7 @@ public class TbCaffeineCacheConfiguration { //SimpleCacheManager is not a bean (will be wrapped), so call initializeCaches manually manager.initializeCaches(); - return new TransactionAwareCacheManagerProxy(manager); + return manager; } private CaffeineCache buildCache(String name, CacheSpecs cacheSpec) { diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java index 92ab0489c7..302b0e8187 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbTransactionalCache.java @@ -39,6 +39,12 @@ public interface TbTransactionalCache newTransactionForKey(K key); + /** + * Note that all keys should be in the same cache slot for redis. You may control the cache slot using '{}' bracers. + * See CLUSTER KEYSLOT command for more details. + * @param keys - list of keys to use + * @return transaction object + */ TbCacheTransaction newTransactionForKeys(List keys); default V getAndPutInTransaction(K key, Supplier dbCall, boolean cacheNullValue) { diff --git a/common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java index 7e65d6a230..f561119f7f 100644 --- a/common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java +++ b/common/cache/src/test/java/org/thingsboard/server/cache/CacheSpecsMapTest.java @@ -21,8 +21,8 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cache.CacheManager; -import org.springframework.cache.transaction.TransactionAwareCacheDecorator; -import org.springframework.cache.transaction.TransactionAwareCacheManagerProxy; +import org.springframework.cache.caffeine.CaffeineCache; +import org.springframework.cache.support.SimpleCacheManager; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit.jupiter.SpringExtension; @@ -45,14 +45,16 @@ public class CacheSpecsMapTest { CacheManager cacheManager; @Test - public void verifyTransactionAwareCacheManagerProxy() { - assertThat(cacheManager).isInstanceOf(TransactionAwareCacheManagerProxy.class); + public void verifyNotTransactionAwareCacheManagerProxy() { + // We no longer use built-in transaction support for the caches, because we have our own cache cleanup and transaction logic that implements CAS. + assertThat(cacheManager).isInstanceOf(SimpleCacheManager.class); } @Test - public void givenCacheConfig_whenCacheManagerReady_thenVerifyExistedCachesWithTransactionAwareCacheDecorator() { - assertThat(cacheManager.getCache("relations")).isInstanceOf(TransactionAwareCacheDecorator.class); - assertThat(cacheManager.getCache("devices")).isInstanceOf(TransactionAwareCacheDecorator.class); + public void givenCacheConfig_whenCacheManagerReady_thenVerifyExistedCachesWithNoTransactionAwareCacheDecorator() { + // We no longer use built-in transaction support for the caches, because we have our own cache cleanup and transaction logic that implements CAS. + assertThat(cacheManager.getCache("relations")).isInstanceOf(CaffeineCache.class); + assertThat(cacheManager.getCache("devices")).isInstanceOf(CaffeineCache.class); } @Test diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 22953f259a..d258aa3820 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -31,8 +31,9 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.queue.TbQueueCallback; import org.thingsboard.server.queue.TbQueueClusterService; @@ -47,9 +48,11 @@ public interface TbClusterService extends TbQueueClusterService { void pushMsgToCore(ToDeviceActorNotificationMsg msg, TbQueueCallback callback); + void pushMsgToVersionControl(TenantId tenantId, ToVersionControlServiceMsg msg, TbQueueCallback callback); + void pushNotificationToCore(String targetServiceId, FromDeviceRpcResponse response, TbQueueCallback callback); - void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, TransportProtos.ToRuleEngineMsg msg, TbQueueCallback callback); + void pushMsgToRuleEngine(TopicPartitionInfo tpi, UUID msgId, ToRuleEngineMsg msg, TbQueueCallback callback); void pushMsgToRuleEngine(TenantId tenantId, EntityId entityId, TbMsg msg, TbQueueCallback callback); @@ -85,5 +88,5 @@ public interface TbClusterService extends TbQueueClusterService { void onEdgeEventUpdate(TenantId tenantId, EdgeId edgeId); - void sendNotificationMsgToEdgeService(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action); + void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action); } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 50f114856f..45cbda66d3 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -454,6 +454,14 @@ message GetOtaPackageResponseMsg { string fileName = 8; } +message DeviceActivityProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; + int64 lastActivityTime = 5; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -618,8 +626,8 @@ message TbSubscriptionUpdateValueListProto { } message TbSubscriptionUpdateTsValue { - int64 ts = 1; - optional string value = 2; + int64 ts = 1; + optional string value = 2; } /** @@ -676,6 +684,193 @@ message EdgeNotificationMsgProto { PostAttributeMsg postAttributesMsg = 12; } +/** + TB Core to Version Control Service + */ +message CommitRequestMsg { + string txId = 1; // To correlate prepare, add, delete and push messages + PrepareMsg prepareMsg = 2; + AddMsg addMsg = 3; + DeleteMsg deleteMsg = 4; + PushMsg pushMsg = 5; + AbortMsg abortMsg = 6; +} + +message CommitResponseMsg { + int64 ts = 1; + string commitId = 2; + string name = 3; + string author = 4; + int32 added = 5; + int32 modified = 6; + int32 removed = 7; +} + +message PrepareMsg { + string commitMsg = 1; + string branchName = 2; + string authorName = 3; + string authorEmail = 4; +} + +message AddMsg { + string relativePath = 1; + string entityDataJson = 2; +} + +message DeleteMsg { + string relativePath = 1; +} + +message PushMsg { +} + +message AbortMsg { +} + +message ListVersionsRequestMsg { + string branchName = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; + int32 pageSize = 5; + int32 page = 6; + string textSearch = 7; + string sortProperty = 8; + string sortDirection = 9; +} + +message EntityVersionProto { + int64 ts = 1; + string id = 2; + string name = 3; + string author = 4; +} + +message ListVersionsResponseMsg { + repeated EntityVersionProto versions = 1; + int32 totalPages = 2; + int64 totalElements = 3; + bool hasNext = 4; +} + +message ListEntitiesRequestMsg { + string branchName = 1; + string versionId = 2; + string entityType = 3; +} + +message VersionedEntityInfoProto { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; +} + +message ListEntitiesResponseMsg { + repeated VersionedEntityInfoProto entities = 1; +} + +message ListBranchesRequestMsg { +} + +message BranchInfoProto { + string name = 1; + bool isDefault = 2; +} + +message ListBranchesResponseMsg { + repeated BranchInfoProto branches = 1; +} + +message EntityContentRequestMsg { + string versionId = 1; + string entityType = 2; + int64 entityIdMSB = 3; + int64 entityIdLSB = 4; +} + +message EntityContentResponseMsg { + string data = 1; +} + +message EntitiesContentRequestMsg { + string versionId = 1; + string entityType = 2; + int32 offset = 3; + int32 limit = 4; +} + +message EntitiesContentResponseMsg { + repeated string data = 1; +} + +message VersionsDiffRequestMsg { + string path = 1; + string versionId1 = 2; + string versionId2 = 3; +} + +message VersionsDiffResponseMsg { + repeated EntityVersionsDiff diff = 1; +} + +message EntityVersionsDiff { + string entityType = 1; + int64 entityIdMSB = 2; + int64 entityIdLSB = 3; + string entityDataAtVersion1 = 4; + string entityDataAtVersion2 = 5; + string rawDiff = 6; +} + +message ContentsDiffRequestMsg { + string content1 = 1; + string content2 = 2; +} + +message ContentsDiffResponseMsg { + string diff = 1; +} + +message GenericRepositoryRequestMsg {} + +message GenericRepositoryResponseMsg {} + +message ToVersionControlServiceMsg { + string nodeId = 1; + int64 tenantIdMSB = 2; + int64 tenantIdLSB = 3; + int64 requestIdMSB = 4; + int64 requestIdLSB = 5; + bytes vcSettings = 6; + GenericRepositoryRequestMsg initRepositoryRequest = 7; + GenericRepositoryRequestMsg testRepositoryRequest = 8; + GenericRepositoryRequestMsg clearRepositoryRequest = 9; + CommitRequestMsg commitRequest = 10; + ListVersionsRequestMsg listVersionRequest = 11; + ListEntitiesRequestMsg listEntitiesRequest = 12; + ListBranchesRequestMsg listBranchesRequest = 13; + EntityContentRequestMsg entityContentRequest = 14; + EntitiesContentRequestMsg entitiesContentRequest = 15; + VersionsDiffRequestMsg versionsDiffRequest = 16; + ContentsDiffRequestMsg contentsDiffRequest = 17; +} + +message VersionControlResponseMsg { + int64 requestIdMSB = 1; + int64 requestIdLSB = 2; + string error = 3; + GenericRepositoryResponseMsg genericResponse = 4; + CommitResponseMsg commitResponse = 5; + ListBranchesResponseMsg listBranchesResponse = 6; + ListEntitiesResponseMsg listEntitiesResponse = 7; + ListVersionsResponseMsg listVersionsResponse = 8; + EntityContentResponseMsg entityContentResponse = 9; + EntitiesContentResponseMsg entitiesContentResponse = 10; + VersionsDiffResponseMsg versionsDiffResponse = 11; + ContentsDiffResponseMsg contentsDiffResponse = 12; +} + /** * Main messages; */ @@ -720,6 +915,7 @@ message ToCoreMsg { SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3; bytes toDeviceActorNotificationMsg = 4; EdgeNotificationMsgProto edgeNotificationMsg = 5; + DeviceActivityProto deviceActivityMsg = 6; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -730,6 +926,7 @@ message ToCoreNotificationMsg { bytes edgeEventUpdateMsg = 4; QueueUpdateMsg queueUpdateMsg = 5; QueueDeleteMsg queueDeleteMsg = 6; + VersionControlResponseMsg vcResponseMsg = 7; } /* Messages that are handled by ThingsBoard RuleEngine Service */ @@ -793,3 +990,5 @@ message ToOtaPackageStateServiceMsg { int64 otaPackageIdLSB = 7; string type = 8; } + + diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java index 84c3f9332f..4032797b5b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java @@ -25,6 +25,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import java.util.List; + public interface DashboardService { Dashboard findDashboardById(TenantId tenantId, DashboardId dashboardId); @@ -64,4 +66,7 @@ public interface DashboardService { PageData findDashboardsByTenantIdAndEdgeId(TenantId tenantId, EdgeId edgeId, PageLink pageLink); DashboardInfo findFirstDashboardInfoByTenantIdAndName(TenantId tenantId, String name); + + List findTenantDashboardsByTitle(TenantId tenantId, String title); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java index ea95923275..c0c72552f6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/edge/EdgeService.java @@ -84,5 +84,5 @@ public interface EdgeService { PageData findRelatedEdgeIdsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); - String findMissingToRelatedRuleChains(TenantId tenantId, EdgeId edgeId); + String findMissingToRelatedRuleChains(TenantId tenantId, EdgeId edgeId, String tbRuleChainInputNodeClassName); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java index bbc011845a..bb790c78cb 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import java.util.List; @@ -72,6 +71,8 @@ public interface EntityViewService { ListenableFuture> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId); + List findEntityViewsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId); + void deleteEntityView(TenantId tenantId, EntityViewId entityViewId); void deleteEntityViewsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index dbd641c1c8..e7df5eea93 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import java.util.Collection; import java.util.List; /** @@ -34,12 +35,16 @@ import java.util.List; */ public interface RelationService { - ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + + boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean saveRelation(TenantId tenantId, EntityRelation relation); + void saveRelations(TenantId tenantId, List relations); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); boolean deleteRelation(TenantId tenantId, EntityRelation relation); @@ -52,8 +57,6 @@ public interface RelationService { void deleteEntityRelations(TenantId tenantId, EntityId entity); - ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entity); - List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); ListenableFuture> findByFromAsync(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java index 3328372f2e..9a856ae66a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java @@ -28,11 +28,11 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainData; import org.thingsboard.server.common.data.rule.RuleChainImportResult; import org.thingsboard.server.common.data.rule.RuleChainMetaData; -import org.thingsboard.server.common.data.rule.RuleChainOutputLabelsUsage; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.common.data.rule.RuleNode; +import java.util.Collection; import java.util.List; /** @@ -66,6 +66,8 @@ public interface RuleChainService { PageData findTenantRuleChainsByType(TenantId tenantId, RuleChainType type, PageLink pageLink); + Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name); + void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId); void deleteRuleChainsByTenantId(TenantId tenantId); @@ -97,4 +99,7 @@ public interface RuleChainService { PageData findAllRuleNodesByType(String type, PageLink pageLink); RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode); + + void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java index d9238c9d7b..82839da1b8 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsService.java @@ -25,6 +25,12 @@ public interface AdminSettingsService { AdminSettings findAdminSettingsByKey(TenantId tenantId, String key); + AdminSettings findAdminSettingsByTenantIdAndKey(TenantId tenantId, String key); + AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings); + boolean deleteAdminSettingsByTenantIdAndKey(TenantId tenantId, String key); + + void deleteAdminSettingsByTenantId(TenantId tenantId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java index fbb9dfa0e1..2dcccc843f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java @@ -35,6 +35,8 @@ public interface TenantService { Tenant saveTenant(Tenant tenant); + boolean tenantExists(TenantId tenantId); + void deleteTenant(TenantId tenantId); PageData findTenants(PageLink pageLink); @@ -44,4 +46,6 @@ public interface TenantService { List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId); void deleteTenants(); + + PageData findTenantsIds(PageLink pageLink); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java index 3b639659c8..d88fabc052 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AdminSettings.java @@ -21,14 +21,17 @@ import org.thingsboard.server.common.data.id.AdminSettingsId; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class AdminSettings extends BaseData { +public class AdminSettings extends BaseData implements HasTenantId { private static final long serialVersionUID = -7670322981725511892L; + private TenantId tenantId; + @NoXss @Length(fieldName = "key") private String key; @@ -44,6 +47,7 @@ public class AdminSettings extends BaseData { public AdminSettings(AdminSettings adminSettings) { super(adminSettings); + this.tenantId = adminSettings.getTenantId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -60,7 +64,16 @@ public class AdminSettings extends BaseData { return super.getCreatedTime(); } - @ApiModelProperty(position = 3, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") + @ApiModelProperty(position = 3, value = "JSON object with Tenant Id.", readOnly = true) + public TenantId getTenantId() { + return tenantId; + } + + public void setTenantId(TenantId tenantId) { + this.tenantId = tenantId; + } + + @ApiModelProperty(position = 4, value = "The Administration Settings key, (e.g. 'general' or 'mail')", example = "mail") public String getKey() { return key; } @@ -69,7 +82,7 @@ public class AdminSettings extends BaseData { this.key = key; } - @ApiModelProperty(position = 4, value = "JSON representation of the Administration Settings value") + @ApiModelProperty(position = 5, value = "JSON representation of the Administration Settings value") public JsonNode getJsonValue() { return jsonValue; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java index c8cf730ead..f2ada352e7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/BaseData.java @@ -65,9 +65,7 @@ public abstract class BaseData extends IdBased implement if (getClass() != obj.getClass()) return false; BaseData other = (BaseData) obj; - if (createdTime != other.createdTime) - return false; - return true; + return createdTime == other.createdTime; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 49db7befd3..c473bcbd6c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -26,10 +26,14 @@ public class CacheConstants { public static final String CLAIM_DEVICES_CACHE = "claimDevices"; public static final String SECURITY_SETTINGS_CACHE = "securitySettings"; public static final String TENANT_PROFILE_CACHE = "tenantProfiles"; + public static final String TENANTS_CACHE = "tenants"; public static final String DEVICE_PROFILE_CACHE = "deviceProfiles"; public static final String ATTRIBUTES_CACHE = "attributes"; public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; public static final String OTA_PACKAGE_CACHE = "otaPackages"; public static final String OTA_PACKAGE_DATA_CACHE = "otaPackagesData"; + public static final String REPOSITORY_SETTINGS_CACHE = "repositorySettings"; + public static final String AUTO_COMMIT_SETTINGS_CACHE = "autoCommitSettings"; public static final String TWO_FA_VERIFICATION_CODES_CACHE = "twoFaVerificationCodes"; + public static final String VERSION_CONTROL_TASK_CACHE = "versionControlTask"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index ab21d70795..9ce638e667 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,12 +20,16 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -public class Customer extends ContactBased implements HasTenantId { +@EqualsAndHashCode(callSuper = true) +public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; @@ -36,6 +40,9 @@ public class Customer extends ContactBased implements HasTenantId { @ApiModelProperty(position = 5, required = true, value = "JSON object with Tenant Id") private TenantId tenantId; + @Getter @Setter + private CustomerId externalId; + public Customer() { super(); } @@ -48,6 +55,7 @@ public class Customer extends ContactBased implements HasTenantId { super(customer); this.tenantId = customer.getTenantId(); this.title = customer.getTitle(); + this.externalId = customer.getExternalId(); } public TenantId getTenantId() { @@ -161,37 +169,6 @@ public class Customer extends ContactBased implements HasTenantId { return getTitle(); } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((tenantId == null) ? 0 : tenantId.hashCode()); - result = prime * result + ((title == null) ? 0 : title.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Customer other = (Customer) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; - } - @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 450c88cbb5..049c830745 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -15,16 +15,33 @@ */ package org.thingsboard.server.common.data; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; -public class Dashboard extends DashboardInfo { +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.StreamSupport; + +@EqualsAndHashCode(callSuper = true) +public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; - + private transient JsonNode configuration; - + + @Getter + @Setter + private DashboardId externalId; + public Dashboard() { super(); } @@ -40,6 +57,7 @@ public class Dashboard extends DashboardInfo { public Dashboard(Dashboard dashboard) { super(dashboard); this.configuration = dashboard.getConfiguration(); + this.externalId = dashboard.getExternalId(); } @ApiModelProperty(position = 9, value = "JSON object with main configuration of the dashboard: layouts, widgets, aliases, etc. " + @@ -54,29 +72,32 @@ public class Dashboard extends DashboardInfo { this.configuration = configuration; } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((configuration == null) ? 0 : configuration.hashCode()); - return result; + @JsonIgnore + public List getEntityAliasesConfig() { + return getChildObjects("entityAliases"); } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Dashboard other = (Dashboard) obj; - if (configuration == null) { - if (other.configuration != null) - return false; - } else if (!configuration.equals(other.configuration)) - return false; - return true; + @JsonIgnore + public List getWidgetsConfig() { + return getChildObjects("widgets"); + } + + @JsonIgnore + private List getChildObjects(String propertyName) { + return Optional.ofNullable(configuration) + .map(config -> config.get(propertyName)) + .filter(node -> !node.isEmpty()) + .map(node -> (ObjectNode) node) + .map(object -> { + List widgets = new ArrayList<>(object.size()); + object.forEach(child -> { + if (child.isObject()) { + widgets.add((ObjectNode) child); + } + }); + return widgets; + }) + .orElse(Collections.emptyList()); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java index e603f68c77..0a6d2260d2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DashboardInfo.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; @@ -26,6 +27,7 @@ import org.thingsboard.server.common.data.validation.NoXss; import javax.validation.Valid; import java.util.HashSet; +import java.util.Objects; import java.util.Set; @ApiModel @@ -63,7 +65,7 @@ public class DashboardInfo extends SearchTextBased implements HasNa @ApiModelProperty(position = 1, value = "JSON object with the dashboard Id. " + "Specify existing dashboard Id to update the dashboard. " + "Referencing non-existing dashboard id will cause error. " + - "Omit this field to create new dashboard." ) + "Omit this field to create new dashboard.") @Override public DashboardId getId() { return super.getId(); @@ -133,7 +135,6 @@ public class DashboardInfo extends SearchTextBased implements HasNa return this.assignedCustomers != null && this.assignedCustomers.contains(new ShortCustomerInfo(customerId, null, false)); } - public ShortCustomerInfo getAssignedCustomerInfo(CustomerId customerId) { if (this.assignedCustomers != null) { for (ShortCustomerInfo customerInfo : this.assignedCustomers) { @@ -201,25 +202,17 @@ public class DashboardInfo extends SearchTextBased implements HasNa } @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - DashboardInfo other = (DashboardInfo) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + DashboardInfo that = (DashboardInfo) o; + return mobileHide == that.mobileHide + && Objects.equals(tenantId, that.tenantId) + && Objects.equals(title, that.title) + && Objects.equals(image, that.image) + && Objects.equals(assignedCustomers, that.assignedCustomers) + && Objects.equals(mobileOrder, that.mobileOrder); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java index 4d7bf5e822..678c68c115 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Device.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Device.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.id.CustomerId; @@ -38,7 +40,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) @Slf4j -public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage { +public class Device extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; @@ -61,6 +63,9 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen private OtaPackageId firmwareId; private OtaPackageId softwareId; + @Getter @Setter + private DeviceId externalId; + public Device() { super(); } @@ -80,6 +85,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setDeviceData(device.getDeviceData()); this.firmwareId = device.getFirmwareId(); this.softwareId = device.getSoftwareId(); + this.externalId = device.getExternalId(); } public Device updateDevice(Device device) { @@ -93,6 +99,7 @@ public class Device extends SearchTextBasedWithAdditionalInfo implemen this.setFirmwareId(device.getFirmwareId()); this.setSoftwareId(device.getSoftwareId()); Optional.ofNullable(device.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.setExternalId(device.getExternalId()); return this; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java index f432862499..e087ba0b7c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java @@ -45,7 +45,7 @@ import static org.thingsboard.server.common.data.SearchTextBasedWithAdditionalIn @ToString(exclude = {"image", "profileDataBytes"}) @EqualsAndHashCode(callSuper = true) @Slf4j -public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage { +public class DeviceProfile extends SearchTextBased implements HasName, HasTenantId, HasOtaPackage, ExportableEntity { private static final long serialVersionUID = 6998485460273302018L; @@ -94,6 +94,8 @@ public class DeviceProfile extends SearchTextBased implements H @ApiModelProperty(position = 10, value = "Reference to the software OTA package. If present, the specified package will be used as default device software. ") private OtaPackageId softwareId; + private DeviceProfileId externalId; + public DeviceProfile() { super(); } @@ -116,6 +118,7 @@ public class DeviceProfile extends SearchTextBased implements H this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey(); this.firmwareId = deviceProfile.getFirmwareId(); this.softwareId = deviceProfile.getSoftwareId(); + this.externalId = deviceProfile.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the device profile Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileInfo.java index a05950d11d..f99d5181f7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfileInfo.java @@ -63,4 +63,9 @@ public class DeviceProfileInfo extends EntityInfo { this.transportType = transportType; } + public DeviceProfileInfo(DeviceProfile profile) { + this(profile.getId(), profile.getName(), profile.getImage(), profile.getDefaultDashboardId(), + profile.getType(), profile.getTransportType()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EdgeUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/EdgeUtils.java index d62efd6fcb..5c8f7fabc9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EdgeUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EdgeUtils.java @@ -60,6 +60,8 @@ public final class EdgeUtils { return EdgeEventType.WIDGETS_BUNDLE; case WIDGET_TYPE: return EdgeEventType.WIDGET_TYPE; + case OTA_PACKAGE: + return EdgeEventType.OTA_PACKAGE; default: log.warn("Unsupported entity type [{}]", entityType); return null; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java index 1f238fd672..a7e21b8a25 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityView.java @@ -36,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @AllArgsConstructor @EqualsAndHashCode(callSuper = true) public class EntityView extends SearchTextBasedWithAdditionalInfo - implements HasName, HasTenantId, HasCustomerId { + implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 5582010124562018986L; @@ -59,6 +59,8 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo @ApiModelProperty(position = 10, value = "Represents the end time of the interval that is used to limit access to target device telemetry. Customer will not be able to see entity telemetry that is outside the specified interval;") private long endTimeMs; + private EntityViewId externalId; + public EntityView() { super(); } @@ -77,6 +79,7 @@ public class EntityView extends SearchTextBasedWithAdditionalInfo this.keys = entityView.getKeys(); this.startTimeMs = entityView.getStartTimeMs(); this.endTimeMs = entityView.getEndTimeMs(); + this.externalId = entityView.getExternalId(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java new file mode 100644 index 0000000000..64834c8b15 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ExportableEntity.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.HasId; +import org.thingsboard.server.common.data.id.TenantId; + +public interface ExportableEntity extends HasId, HasName { + + void setId(I id); + + I getExternalId(); + void setExternalId(I externalId); + + long getCreatedTime(); + void setCreatedTime(long createdTime); + + void setTenantId(TenantId tenantId); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 17f3713b74..ff3d3c0fd4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -19,6 +19,10 @@ import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { + public static final String EMPTY = ""; + + public static final int INDEX_NOT_FOUND = -1; + public static boolean isEmpty(String source) { return source == null || source.isEmpty(); } @@ -35,6 +39,44 @@ public class StringUtils { return source != null && !source.isEmpty() && !source.trim().isEmpty(); } + public static String removeStart(final String str, final String remove) { + if (isEmpty(str) || isEmpty(remove)) { + return str; + } + if (str.startsWith(remove)){ + return str.substring(remove.length()); + } + return str; + } + + public static String substringBefore(final String str, final String separator) { + if (isEmpty(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringBetween(final String str, final String open, final String close) { + if (str == null || open == null || close == null) { + return null; + } + final int start = str.indexOf(open); + if (start != INDEX_NOT_FOUND) { + final int end = str.indexOf(close, start + open.length()); + if (end != INDEX_NOT_FOUND) { + return str.substring(start + open.length(), end); + } + } + return null; + } + public static String obfuscate(String input, int seenMargin, char obfuscationChar, int startIndexInclusive, int endIndexExclusive) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java index db68df4901..da6d9615a1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/asset/Asset.java @@ -19,6 +19,9 @@ import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -33,7 +36,7 @@ import java.util.Optional; @ApiModel @EqualsAndHashCode(callSuper = true) -public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId { +public class Asset extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, HasCustomerId, ExportableEntity { private static final long serialVersionUID = 2807343040519543363L; @@ -49,6 +52,9 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements @Length(fieldName = "label") private String label; + @Getter @Setter + private AssetId externalId; + public Asset() { super(); } @@ -64,6 +70,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.name = asset.getName(); this.type = asset.getType(); this.label = asset.getLabel(); + this.externalId = asset.getExternalId(); } public void update(Asset asset) { @@ -73,6 +80,7 @@ public class Asset extends SearchTextBasedWithAdditionalInfo implements this.type = asset.getType(); this.label = asset.getLabel(); Optional.ofNullable(asset.getAdditionalInfo()).ifPresent(this::setAdditionalInfo); + this.externalId = asset.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the asset Id. " + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java index 3b1489d05f..cd650eb6c3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java @@ -31,5 +31,6 @@ public enum EdgeEventType { TENANT, WIDGETS_BUNDLE, WIDGET_TYPE, - ADMIN_SETTINGS + ADMIN_SETTINGS, + OTA_PACKAGE } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 8bd13b7c3f..f5e3fdd6f3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -107,6 +107,8 @@ public class EntityIdFactory { return new WidgetsBundleId(uuid); case WIDGET_TYPE: return new WidgetTypeId(uuid); + case OTA_PACKAGE: + return new OtaPackageId(uuid); case EDGE: return new EdgeId(uuid); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java index cfe6918a7d..4682c8f95b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/IdBased.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.id; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonSetter; import java.io.Serializable; import java.util.UUID; @@ -33,6 +34,7 @@ public abstract class IdBased implements HasId { this.id = id; } + @JsonSetter public void setId(I id) { this.id = id; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java index 17d7ffef0b..eca9e331cd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleChain.java @@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; @@ -35,7 +36,7 @@ import org.thingsboard.server.common.data.validation.NoXss; @Data @EqualsAndHashCode(callSuper = true) @Slf4j -public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { +public class RuleChain extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId, ExportableEntity { private static final long serialVersionUID = -5656679015121935465L; @@ -56,6 +57,8 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im @ApiModelProperty(position = 9, value = "Reserved for future usage. The actual list of rule nodes and their relations is stored in the database separately.") private transient JsonNode configuration; + private RuleChainId externalId; + @JsonIgnore private byte[] configurationBytes; @@ -75,6 +78,7 @@ public class RuleChain extends SearchTextBasedWithAdditionalInfo im this.firstRuleNodeId = ruleChain.getFirstRuleNodeId(); this.root = ruleChain.isRoot(); this.setConfiguration(ruleChain.getConfiguration()); + this.setExternalId(ruleChain.getExternalId()); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java index fa2760dbc3..d9dce57f46 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/rule/RuleNode.java @@ -53,6 +53,8 @@ public class RuleNode extends SearchTextBasedWithAdditionalInfo impl @JsonIgnore private byte[] configurationBytes; + private RuleNodeId externalId; + public RuleNode() { super(); } @@ -68,6 +70,7 @@ public class RuleNode extends SearchTextBasedWithAdditionalInfo impl this.name = ruleNode.getName(); this.debugMode = ruleNode.isDebugMode(); this.setConfiguration(ruleNode.getConfiguration()); + this.externalId = ruleNode.getExternalId(); } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java new file mode 100644 index 0000000000..0e2929bef8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync; + +import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +@JacksonAnnotationsInside +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = JsonTypeInfo.As.EXTERNAL_PROPERTY) +@JsonSubTypes({ + @Type(name = "DEVICE", value = Device.class), + @Type(name = "RULE_CHAIN", value = RuleChain.class), + @Type(name = "DEVICE_PROFILE", value = DeviceProfile.class), + @Type(name = "ASSET", value = Asset.class), + @Type(name = "DASHBOARD", value = Dashboard.class), + @Type(name = "CUSTOMER", value = Customer.class), + @Type(name = "ENTITY_VIEW", value = EntityView.class), + @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundle.class) +}) +public @interface JsonTbEntity { +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java new file mode 100644 index 0000000000..1cb2ac8c74 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync; + +import org.thingsboard.server.common.data.exception.ThingsboardException; + +public interface ThrowingRunnable { + + void run() throws ThingsboardException; + + default ThrowingRunnable andThen(ThrowingRunnable after) { + return () -> { + this.run(); + after.run(); + }; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java new file mode 100644 index 0000000000..dcf69c0877 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; + +@Data +public class AttributeExportData { + private String key; + private Long lastUpdateTs; + + private Boolean booleanValue; + private String strValue; + private Long longValue; + private Double doubleValue; + private String jsonValue; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java new file mode 100644 index 0000000000..27d16899d1 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class DeviceExportData extends EntityExportData { + + @JsonProperty(index = 3) + @JsonIgnoreProperties({"id", "deviceId", "createdTime"}) + private DeviceCredentials credentials; + + @JsonIgnore + @Override + public boolean hasCredentials() { + return credentials != null; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java new file mode 100644 index 0000000000..afe6c925c7 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -0,0 +1,98 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.sync.JsonTbEntity; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) +@JsonSubTypes({ + @Type(name = "DEVICE", value = DeviceExportData.class), + @Type(name = "RULE_CHAIN", value = RuleChainExportData.class), + @Type(name = "WIDGETS_BUNDLE", value = WidgetsBundleExportData.class) +}) +@JsonInclude(JsonInclude.Include.NON_NULL) +@Data +public class EntityExportData> { + + public static final Comparator relationsComparator = Comparator + .comparing(EntityRelation::getFrom, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTo, Comparator.comparing(EntityId::getId)) + .thenComparing(EntityRelation::getTypeGroup) + .thenComparing(EntityRelation::getType); + + public static final Comparator attrComparator = Comparator + .comparing(AttributeExportData::getKey).thenComparing(AttributeExportData::getLastUpdateTs); + + @JsonProperty(index = 2) + @JsonIgnoreProperties({"tenantId", "createdTime"}) + @JsonTbEntity + private E entity; + @JsonProperty(index = 1) + private EntityType entityType; + + @JsonProperty(index = 100) + private List relations; + @JsonProperty(index = 101) + private Map> attributes; + + public EntityExportData sort() { + if (relations != null && !relations.isEmpty()) { + relations.sort(relationsComparator); + } + if (attributes != null && !attributes.isEmpty()) { + attributes.values().forEach(list -> list.sort(attrComparator)); + } + return this; + } + + @JsonIgnore + public EntityId getExternalId() { + return entity.getExternalId() != null ? entity.getExternalId() : entity.getId(); + } + + @JsonIgnore + public boolean hasCredentials() { + return false; + } + + @JsonIgnore + public boolean hasAttributes() { + return attributes != null; + } + + @JsonIgnore + public boolean hasRelations() { + return relations != null; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java new file mode 100644 index 0000000000..1a625640e0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityExportSettings { + private boolean exportRelations; + private boolean exportAttributes; + private boolean exportCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java new file mode 100644 index 0000000000..1c6f4c060a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; + +@Data +public class EntityImportResult> { + + private E savedEntity; + private E oldEntity; + private EntityType entityType; + + private ThrowingRunnable saveReferencesCallback = () -> {}; + private ThrowingRunnable sendEventsCallback = () -> {}; + + private boolean updatedAllExternalIds = true; + + private boolean created; + private boolean updated; + private boolean updatedRelatedEntities; + + public void addSaveReferencesCallback(ThrowingRunnable callback) { + this.saveReferencesCallback = this.saveReferencesCallback.andThen(callback); + } + + public void addSendEventsCallback(ThrowingRunnable callback) { + this.sendEventsCallback = this.sendEventsCallback.andThen(callback); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java new file mode 100644 index 0000000000..9b95b8eca2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityImportSettings { + private boolean findExistingByName; + private boolean updateRelations; + private boolean saveAttributes; + private boolean saveCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java new file mode 100644 index 0000000000..5c37aa35b6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; + +@EqualsAndHashCode(callSuper = true) +@ToString(callSuper = true) +@Data +public class RuleChainExportData extends EntityExportData { + + @JsonProperty(index = 3) + @JsonIgnoreProperties("ruleChainId") + private RuleChainMetaData metaData; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java new file mode 100644 index 0000000000..aa7600a48d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/WidgetsBundleExportData.java @@ -0,0 +1,42 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.widget.BaseWidgetType; +import org.thingsboard.server.common.data.widget.WidgetTypeDetails; +import org.thingsboard.server.common.data.widget.WidgetsBundle; + +import java.util.Comparator; +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +public class WidgetsBundleExportData extends EntityExportData { + + @JsonProperty(index = 3) + private List widgets; + + @Override + public EntityExportData sort() { + super.sort(); + widgets.sort(Comparator.comparing(BaseWidgetType::getAlias)); + return this; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java new file mode 100644 index 0000000000..2120ec4c44 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/AutoCommitSettings.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig; + +import java.util.HashMap; + +public class AutoCommitSettings extends HashMap { + + private static final long serialVersionUID = -5757067601838792059L; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java new file mode 100644 index 0000000000..fcf1f4470e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; + +import java.util.Objects; + +@Data +public class BranchInfo { + private final String name; + private final boolean isDefault; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BranchInfo that = (BranchInfo) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java new file mode 100644 index 0000000000..800583b0b8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +public class EntityDataDiff { + private EntityExportData currentVersion; + private EntityExportData otherVersion; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java new file mode 100644 index 0000000000..2529ec8af6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataInfo.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityDataInfo { + boolean hasRelations; + boolean hasAttributes; + boolean hasCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java new file mode 100644 index 0000000000..bc3475c89e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; + +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class EntityLoadError { + + private String type; + private EntityId source; + private EntityId target; + private String message; + + public static EntityLoadError credentialsError(EntityId sourceId) { + return EntityLoadError.builder().type("DEVICE_CREDENTIALS_CONFLICT").source(sourceId).build(); + } + + public static EntityLoadError referenceEntityError(EntityId sourceId, EntityId targetId) { + return EntityLoadError.builder().type("MISSING_REFERENCED_ENTITY").source(sourceId).target(targetId).build(); + } + + public static EntityLoadError runtimeError(String msg) { + return EntityLoadError.builder().type("RUNTIME").message(msg).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java new file mode 100644 index 0000000000..84a28d0770 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityTypeLoadResult { + private EntityType entityType; + private int created; + private int updated; + private int deleted; + + public EntityTypeLoadResult(EntityType entityType) { + this.entityType = entityType; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java new file mode 100644 index 0000000000..e7ef81a562 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class EntityVersion implements Serializable { + + private static final long serialVersionUID = -3705022663019175258L; + + private long timestamp; + private String id; + private String name; + private String author; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java new file mode 100644 index 0000000000..2104472047 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersionsDiff.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionsDiff { + private EntityId externalId; + private EntityExportData entityDataAtVersion1; + private EntityExportData entityDataAtVersion2; + private String rawDiff; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java new file mode 100644 index 0000000000..3c0e5849a3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositoryAuthMethod.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +public enum RepositoryAuthMethod { + USERNAME_PASSWORD, + PRIVATE_KEY +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java new file mode 100644 index 0000000000..50e23a1dba --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/RepositorySettings.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.io.Serializable; + +@Data +public class RepositorySettings implements Serializable { + private static final long serialVersionUID = -3211552851889198721L; + + private String repositoryUri; + private RepositoryAuthMethod authMethod; + private String username; + private String password; + private String privateKeyFileName; + private String privateKey; + private String privateKeyPassword; + private String defaultBranch; + + public RepositorySettings() { + } + + public RepositorySettings(RepositorySettings settings) { + this.repositoryUri = settings.getRepositoryUri(); + this.authMethod = settings.getAuthMethod(); + this.username = settings.getUsername(); + this.password = settings.getPassword(); + this.privateKeyFileName = settings.getPrivateKeyFileName(); + this.privateKey = settings.getPrivateKey(); + this.privateKeyPassword = settings.getPrivateKeyPassword(); + this.defaultBranch = settings.getDefaultBranch(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java new file mode 100644 index 0000000000..608cc5ddd3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.io.Serializable; + +@Data +@NoArgsConstructor +public class VersionCreationResult implements Serializable { + private static final long serialVersionUID = 8032189124530267838L; + + private EntityVersion version; + private int added; + private int modified; + private int removed; + + private String error; + private boolean done; + + + public VersionCreationResult(EntityVersion version, int added, int modified, int removed) { + this.version = version; + this.added = added; + this.modified = modified; + this.removed = removed; + this.done = true; + } + + public VersionCreationResult(String error) { + this.error = error; + this.done = true; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java new file mode 100644 index 0000000000..05d96f10ce --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Builder; +import lombok.Data; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; + +@Data +@Builder +@JsonInclude(JsonInclude.Include.NON_NULL) +public class VersionLoadResult implements Serializable { + + private static final long serialVersionUID = -1386093599856747449L; + + private List result; + private EntityLoadError error; + private boolean done; + + public static VersionLoadResult empty() { + return VersionLoadResult.builder().result(Collections.emptyList()).build(); + } + + public static VersionLoadResult success(List result) { + return VersionLoadResult.builder().result(result).build(); + } + + public static VersionLoadResult success(EntityTypeLoadResult result) { + return VersionLoadResult.builder().result(List.of(result)).build(); + } + + public static VersionLoadResult error(EntityLoadError error) { + return VersionLoadResult.builder().error(error).done(true).build(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java new file mode 100644 index 0000000000..fd278cde1f --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@NoArgsConstructor +public class VersionedEntityInfo { + private EntityId externalId; + // etc.. + + public VersionedEntityInfo(EntityId externalId) { + this.externalId = externalId; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java new file mode 100644 index 0000000000..084b2a1059 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/AutoVersionCreateConfig.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class AutoVersionCreateConfig extends VersionCreateConfig { + + private static final long serialVersionUID = 8245450889383315551L; + + private String branch; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java new file mode 100644 index 0000000000..f8d7279f40 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class ComplexVersionCreateRequest extends VersionCreateRequest { + + // Default sync strategy + private SyncStrategy syncStrategy; + private Map entityTypes; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.COMPLEX; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java new file mode 100644 index 0000000000..92cab96354 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; +import java.util.UUID; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionCreateConfig extends VersionCreateConfig { + + //optional + private SyncStrategy syncStrategy; + private List entityIds; + private boolean allEntities; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java new file mode 100644 index 0000000000..507afd116e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionCreateRequest extends VersionCreateRequest { + + private EntityId entityId; + private VersionCreateConfig config; + + @Override + public VersionCreateRequestType getType() { + return VersionCreateRequestType.SINGLE_ENTITY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java new file mode 100644 index 0000000000..baf24efeb8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +public enum SyncStrategy { + MERGE, + OVERWRITE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java new file mode 100644 index 0000000000..86154bdfa0 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class VersionCreateConfig implements Serializable { + private static final long serialVersionUID = 1223723167716612772L; + + private boolean saveRelations; + private boolean saveAttributes; + private boolean saveCredentials; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java new file mode 100644 index 0000000000..9a23838921 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), + @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class) +}) +@Data +public abstract class VersionCreateRequest { + + private String versionName; + private String branch; + + public abstract VersionCreateRequestType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java new file mode 100644 index 0000000000..d57363cf7d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.create; + +public enum VersionCreateRequestType { + SINGLE_ENTITY, + COMPLEX +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java new file mode 100644 index 0000000000..ffb1f25d3c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadConfig extends VersionLoadConfig { + + private boolean removeOtherEntities; + private boolean findExistingEntityByName; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java new file mode 100644 index 0000000000..f2cb83f0d4 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.EntityType; + +import java.util.Map; + +@Data +@EqualsAndHashCode(callSuper = true) +public class EntityTypeVersionLoadRequest extends VersionLoadRequest { + + private Map entityTypes; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.ENTITY_TYPE; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java new file mode 100644 index 0000000000..cf31317b4a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.id.EntityId; + +@Data +@EqualsAndHashCode(callSuper = true) +public class SingleEntityVersionLoadRequest extends VersionLoadRequest { + + private EntityId externalEntityId; + + private VersionLoadConfig config; + + @Override + public VersionLoadRequestType getType() { + return VersionLoadRequestType.SINGLE_ENTITY; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java new file mode 100644 index 0000000000..a27cf06538 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import lombok.Data; + +@Data +public class VersionLoadConfig { + + private boolean loadRelations; + private boolean loadAttributes; + private boolean loadCredentials; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java new file mode 100644 index 0000000000..d9d1329c8b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.Data; + +import static com.fasterxml.jackson.annotation.JsonSubTypes.Type; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") +@JsonSubTypes({ + @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionLoadRequest.class), + @Type(name = "ENTITY_TYPE", value = EntityTypeVersionLoadRequest.class) +}) +@Data +public abstract class VersionLoadRequest { + + private String branch; + private String versionId; + + public abstract VersionLoadRequestType getType(); + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java new file mode 100644 index 0000000000..190e6871d5 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java @@ -0,0 +1,21 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.vc.request.load; + +public enum VersionLoadRequestType { + SINGLE_ENTITY, + ENTITY_TYPE +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java index 1cd49a6729..e7d6383540 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/DefaultTenantProfileConfiguration.java @@ -15,8 +15,6 @@ */ package org.thingsboard.server.common.data.tenant.profile; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -46,6 +44,9 @@ public class DefaultTenantProfileConfiguration implements TenantProfileConfigura private String transportDeviceTelemetryMsgRateLimit; private String transportDeviceTelemetryDataPointsRateLimit; + private String tenantEntityExportRateLimit; + private String tenantEntityImportRateLimit; + private long maxTransportMessages; private long maxTransportDataPoints; private long maxREExecutions; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java index 1515aceb62..4f3de71512 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetsBundle.java @@ -15,10 +15,13 @@ */ package org.thingsboard.server.common.data.widget; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.SearchTextBased; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +30,8 @@ import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; @ApiModel -public class WidgetsBundle extends SearchTextBased implements HasTenantId { +@EqualsAndHashCode(callSuper = true) +public class WidgetsBundle extends SearchTextBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -7627368878362410489L; @@ -63,6 +67,10 @@ public class WidgetsBundle extends SearchTextBased implements H @ApiModelProperty(position = 7, value = "Description", readOnly = true) private String description; + @Getter + @Setter + private WidgetsBundleId externalId; + public WidgetsBundle() { super(); } @@ -78,6 +86,7 @@ public class WidgetsBundle extends SearchTextBased implements H this.title = widgetsBundle.getTitle(); this.image = widgetsBundle.getImage(); this.description = widgetsBundle.getDescription(); + this.externalId = widgetsBundle.getExternalId(); } @ApiModelProperty(position = 1, value = "JSON object with the Widget Bundle Id. " + @@ -100,31 +109,10 @@ public class WidgetsBundle extends SearchTextBased implements H return getTitle(); } + @JsonIgnore @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + (tenantId != null ? tenantId.hashCode() : 0); - result = 31 * result + (alias != null ? alias.hashCode() : 0); - result = 31 * result + (title != null ? title.hashCode() : 0); - result = 31 * result + (image != null ? image.hashCode() : 0); - result = 31 * result + (description != null ? description.hashCode() : 0); - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - WidgetsBundle that = (WidgetsBundle) o; - - if (tenantId != null ? !tenantId.equals(that.tenantId) : that.tenantId != null) return false; - if (alias != null ? !alias.equals(that.alias) : that.alias != null) return false; - if (title != null ? !title.equals(that.title) : that.title != null) return false; - if (image != null ? !image.equals(that.image) : that.image != null) return false; - if (description != null ? !description.equals(that.description) : that.description != null) return false; - return true; + public String getName() { + return title; } @Override @@ -133,7 +121,6 @@ public class WidgetsBundle extends SearchTextBased implements H sb.append("tenantId=").append(tenantId); sb.append(", alias='").append(alias).append('\''); sb.append(", title='").append(title).append('\''); - sb.append(", image='").append(image).append('\''); sb.append(", description='").append(description).append('\''); sb.append('}'); return sb.toString(); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 2bc6a18d6e..218822bf84 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -421,6 +421,26 @@ enum EdgeEntityType { ASSET = 1; } +message OtaPackageUpdateMsg { + UpdateMsgType msgType = 1; + int64 idMSB = 2; + int64 idLSB = 3; + int64 deviceProfileIdMSB = 4; + int64 deviceProfileIdLSB = 5; + string type = 6; + string title = 7; + string version = 8; + string tag = 9; + optional string url = 10; + optional string fileName = 11; + optional string contentType = 12; + optional string checksumAlgorithm = 13; + optional string checksum = 14; + optional int64 dataSize = 15; + optional bytes data = 16; + optional string additionalInfo = 17; +} + /** * Main Messages; */ @@ -477,5 +497,6 @@ message DownlinkMsg { repeated WidgetTypeUpdateMsg widgetTypeUpdateMsg = 19; repeated AdminSettingsUpdateMsg adminSettingsUpdateMsg = 20; repeated DeviceRpcCallMsg deviceRpcCallMsg = 21; + repeated OtaPackageUpdateMsg otaPackageUpdateMsg = 22; } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java new file mode 100644 index 0000000000..2bcba593e4 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.plugin; + +public interface ComponentLifecycleListener { + void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg); +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java index 1c05ac7f66..7f276e7e3f 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceType.java @@ -17,7 +17,7 @@ package org.thingsboard.server.common.msg.queue; public enum ServiceType { - TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR; + TB_CORE, TB_RULE_ENGINE, TB_TRANSPORT, JS_EXECUTOR, TB_VC_EXECUTOR; public static ServiceType of(String serviceType) { return ServiceType.valueOf(serviceType.replace("-", "_").toUpperCase()); diff --git a/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java b/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java index c8583527f1..cf81690826 100644 --- a/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java +++ b/common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java @@ -15,8 +15,11 @@ */ package org.thingsboard.server.common.msg.tools; +import org.awaitility.pollinterval.FixedPollInterval; +import org.awaitility.pollinterval.PollInterval; import org.junit.Test; +import java.time.Duration; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -39,10 +42,11 @@ public class RateLimitsTest { assertThat(rateLimits.tryConsume()).as("new token is available").isFalse(); int expectedRefillTime = (int) (((double) period / capacity) * 1000); - int gap = 100; + int gap = 500; for (int i = 0; i < capacity; i++) { await("token refill for rate limit " + rateLimitConfig) + .pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS)) .atLeast(expectedRefillTime - gap, TimeUnit.MILLISECONDS) .atMost(expectedRefillTime + gap, TimeUnit.MILLISECONDS) .untilAsserted(() -> { @@ -70,6 +74,7 @@ public class RateLimitsTest { int gap = 500; await("tokens refill for rate limit " + rateLimitConfig) + .pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS)) .atLeast(expectedRefillTime - gap, TimeUnit.MILLISECONDS) .atMost(expectedRefillTime + gap, TimeUnit.MILLISECONDS) .untilAsserted(() -> { diff --git a/common/pom.xml b/common/pom.xml index 236fb50a19..62bb46e81c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,7 @@ cache coap-server edge-api + version-control diff --git a/common/queue/pom.xml b/common/queue/pom.xml index e2197d566f..a625ec6e93 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,6 +120,10 @@ com.google.protobuf protobuf-java-util + + de.ruedigermoeller + fst + org.apache.curator curator-recipes diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java index 9fa91a4e40..b87049b634 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java @@ -37,7 +37,8 @@ public class TbServiceBusQueueConfigs { private String notificationsProperties; @Value("${queue.service-bus.queue-properties.js-executor}") private String jsExecutorProperties; - + @Value("${queue.service-bus.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreConfigs; @Getter @@ -48,6 +49,8 @@ public class TbServiceBusQueueConfigs { private Map notificationsConfigs; @Getter private Map jsExecutorConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -56,6 +59,7 @@ public class TbServiceBusQueueConfigs { transportApiConfigs = getConfigs(transportApiProperties); notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e38677b2ce..9fbe9b1104 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate new ArrayList<>()).add(instance); } }); - } else if (ServiceType.TB_CORE.equals(serviceType)) { + } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java index a2d5982421..e43da75869 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java @@ -53,4 +53,11 @@ public class QueueKey { this.queueName = MAIN; this.tenantId = TenantId.SYS_TENANT_ID; } + + @Override + public String toString() { + return "QK(" + queueName + "," + type + "," + + (TenantId.SYS_TENANT_ID.equals(tenantId) ? "system" : tenantId) + + ')'; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java index 31bad5477e..7b5c142c23 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java @@ -40,7 +40,7 @@ public abstract class TbApplicationEventListener i } finally { seqNumberLock.unlock(); } - if (validUpdate) { + if (validUpdate && filterTbApplicationEvent(event)) { onTbApplicationEvent(event); } else { log.info("Application event ignored due to invalid sequence number ({} > {}). Event: {}", lastProcessedSequenceNumber, event.getSequenceNumber(), event); @@ -49,5 +49,8 @@ public abstract class TbApplicationEventListener i protected abstract void onTbApplicationEvent(T event); + protected boolean filterTbApplicationEvent(T event) { + return true; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 89eda16cad..5976a1372b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,7 +33,6 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 4c9194b675..75d0bd9b2d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -40,6 +40,9 @@ public class TbKafkaTopicConfigs { private String jsExecutorProperties; @Value("${queue.kafka.topic-properties.ota-updates:}") private String fwUpdatesProperties; + @Value("${queue.kafka.topic-properties.version-control:}") + private String vcProperties; + @Getter private Map coreConfigs; @@ -53,6 +56,8 @@ public class TbKafkaTopicConfigs { private Map jsExecutorConfigs; @Getter private Map fwUpdatesConfigs; + @Getter + private Map vcConfigs; @PostConstruct private void init() { @@ -62,6 +67,7 @@ public class TbKafkaTopicConfigs { notificationsConfigs = getConfigs(notificationsProperties); jsExecutorConfigs = getConfigs(jsExecutorProperties); fwUpdatesConfigs = getConfigs(fwUpdatesProperties); + vcConfigs = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 660173c71d..2ca5090aed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -46,6 +47,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -57,7 +59,7 @@ import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='monolith'") -public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; @@ -66,6 +68,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbAwsSqsSettings sqsSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueAdmin coreAdmin; @@ -73,6 +76,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public AwsSqsMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -80,6 +84,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbAwsSqsSettings sqsSettings, + TbQueueVersionControlSettings vcSettings, TbAwsSqsQueueAttributes sqsQueueAttributes, TbQueueRemoteJsInvokeSettings jsInvokeSettings) { this.notificationsTopicService = notificationsTopicService; @@ -89,6 +94,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.sqsSettings = sqsSettings; + this.vcSettings = vcSettings; this.jsInvokeSettings = jsInvokeSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); @@ -96,6 +102,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); } @Override @@ -123,6 +130,13 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbAwsSqsConsumerTemplate<>(ruleEngineAdmin, sqsSettings, configuration.getTopic(), @@ -205,6 +219,11 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -222,5 +241,8 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 81736c2b55..89b45e2822 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..d76b3f595d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; +import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; +import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; +import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes; +import org.thingsboard.server.queue.sqs.TbAwsSqsSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs' && '${service.type:null}'=='tb-vc-executor'") +public class AwsSqsTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbAwsSqsSettings sqsSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public AwsSqsTbVersionControlQueueFactory(TbAwsSqsSettings sqsSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbAwsSqsQueueAttributes sqsQueueAttributes + ) { + this.sqsSettings = sqsSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); + this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbAwsSqsProducerTemplate<>(notificationAdmin, sqsSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fe87fc2886..815e0efdf8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -37,28 +37,32 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; @Slf4j @Component @ConditionalOnExpression("'${queue.type:null}'=='in-memory' && '${service.type:null}'=='monolith'") -public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRuleEngineSettings ruleEngineSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, + TbQueueVersionControlSettings vcSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, InMemoryStorage storage) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; + this.vcSettings = vcSettings; this.serviceInfoProvider = serviceInfoProvider; this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; @@ -91,6 +95,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, vcSettings.getTopic()); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new InMemoryTbQueueConsumer<>(storage, configuration.getTopic()); @@ -146,6 +155,11 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, vcSettings.getTopic()); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 104c531e12..b24ae700bb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,6 +52,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -58,7 +60,7 @@ import java.util.concurrent.atomic.AtomicLong; @Component @ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='monolith'") -public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbKafkaSettings kafkaSettings; @@ -68,6 +70,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueAdmin coreAdmin; @@ -76,6 +79,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; + private final AtomicLong consumerCount = new AtomicLong(); public KafkaMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -85,6 +90,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbQueueTransportApiSettings transportApiSettings, TbQueueTransportNotificationSettings transportNotificationSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbKafkaTopicConfigs kafkaTopicConfigs) { this.notificationsTopicService = notificationsTopicService; @@ -95,6 +101,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); @@ -103,6 +110,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -155,6 +163,19 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("monolith-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("monolith-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { String queueName = configuration.getName(); @@ -311,6 +332,16 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("monolith-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -331,5 +362,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 58dec0792a..f476a068d2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -28,6 +28,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -50,6 +51,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @@ -65,6 +67,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbKafkaConsumerStatsService consumerStatsService; private final TbQueueTransportNotificationSettings transportNotificationSettings; @@ -74,6 +77,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin vcAdmin; public KafkaTbCoreQueueFactory(NotificationsTopicService notificationsTopicService, TbKafkaSettings kafkaSettings, @@ -82,6 +86,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbQueueRuleEngineSettings ruleEngineSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbKafkaConsumerStatsService consumerStatsService, TbQueueTransportNotificationSettings transportNotificationSettings, TbKafkaTopicConfigs kafkaTopicConfigs) { @@ -92,6 +97,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.consumerStatsService = consumerStatsService; this.transportNotificationSettings = transportNotificationSettings; @@ -101,6 +107,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.transportApiAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); } @Override @@ -282,6 +289,16 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-core-vc-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(vcSettings.getTopic()); + requestBuilder.admin(vcAdmin); + return requestBuilder.build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -302,5 +319,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { if (fwUpdatesAdmin != null) { fwUpdatesAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..598f86d0f4 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.kafka.TbKafkaAdmin; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; +import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaProducerTemplate; +import org.thingsboard.server.queue.kafka.TbKafkaSettings; +import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='kafka' && '${service.type:null}'=='tb-vc-executor'") +public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbKafkaSettings kafkaSettings; + private final TbServiceInfoProvider serviceInfoProvider; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + private final TbKafkaConsumerStatsService consumerStatsService; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin notificationAdmin; + + public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, + TbServiceInfoProvider serviceInfoProvider, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbKafkaConsumerStatsService consumerStatsService, + TbKafkaTopicConfigs kafkaTopicConfigs) { + this.kafkaSettings = kafkaSettings; + this.serviceInfoProvider = serviceInfoProvider; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + this.consumerStatsService = consumerStatsService; + + this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); + this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + } + + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-to-core-notifications-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getTopic()); + requestBuilder.admin(notificationAdmin); + return requestBuilder.build(); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(vcSettings.getTopic()); + consumerBuilder.clientId("tb-vc-consumer-" + serviceInfoProvider.getServiceId()); + consumerBuilder.groupId("tb-vc-node"); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + consumerBuilder.admin(vcAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + TbKafkaProducerTemplate.TbKafkaProducerTemplateBuilder> requestBuilder = TbKafkaProducerTemplate.builder(); + requestBuilder.settings(kafkaSettings); + requestBuilder.clientId("tb-vc-us-producer-" + serviceInfoProvider.getServiceId()); + requestBuilder.defaultTopic(coreSettings.getUsageStatsTopic()); + requestBuilder.admin(coreAdmin); + return requestBuilder.build(); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index aceb9d4f0d..d439bb0ed7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,13 +52,14 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='monolith'") -public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final TbPubSubSettings pubSubSettings; private final TbQueueCoreSettings coreSettings; @@ -67,12 +69,14 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final NotificationsTopicService notificationsTopicService; private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public PubSubMonolithQueueFactory(TbPubSubSettings pubSubSettings, TbQueueCoreSettings coreSettings, @@ -82,7 +86,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng NotificationsTopicService notificationsTopicService, TbServiceInfoProvider serviceInfoProvider, TbPubSubSubscriptionSettings pubSubSubscriptionSettings, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings) { this.pubSubSettings = pubSubSettings; this.coreSettings = coreSettings; this.ruleEngineSettings = ruleEngineSettings; @@ -90,12 +95,15 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.transportNotificationSettings = transportNotificationSettings; this.notificationsTopicService = notificationsTopicService; this.serviceInfoProvider = serviceInfoProvider; + this.vcSettings = vcSettings; this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); this.ruleEngineAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings()); this.jsExecutorAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getJsExecutorSettings()); this.transportApiAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getTransportApiSettings()); this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + this.vcAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getVcSettings()); + this.jsInvokeSettings = jsInvokeSettings; } @@ -125,6 +133,13 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbPubSubConsumerTemplate<>(ruleEngineAdmin, pubSubSettings, configuration.getTopic(), @@ -207,6 +222,11 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -224,5 +244,8 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 18a6668581..6cf9aa9a45 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..45e71b2299 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; +import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; +import org.thingsboard.server.queue.pubsub.TbPubSubSettings; +import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='pubsub' && '${service.type:null}'=='tb-vc-executor'") +public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbPubSubSettings pubSubSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public PubSubTbVersionControlQueueFactory(TbPubSubSettings pubSubSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbPubSubSubscriptionSettings pubSubSubscriptionSettings + ) { + this.pubSubSettings = pubSubSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getCoreSettings()); + this.notificationAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getNotificationsSettings()); + this.vcAdmin = new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getVcSettings()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbPubSubProducerTemplate<>(notificationAdmin, pubSubSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbPubSubConsumerTemplate<>(vcAdmin, pubSubSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index 2a724289bb..3223e46cc5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsRequest; import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -51,13 +52,14 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='monolith'") -public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; @@ -67,12 +69,14 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbRabbitMqSettings rabbitMqSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public RabbitMqMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -81,7 +85,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE TbQueueTransportNotificationSettings transportNotificationSettings, TbRabbitMqSettings rabbitMqSettings, TbRabbitMqQueueArguments queueArguments, - TbQueueRemoteJsInvokeSettings jsInvokeSettings) { + TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; this.serviceInfoProvider = serviceInfoProvider; @@ -90,12 +95,14 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.transportNotificationSettings = transportNotificationSettings; this.rabbitMqSettings = rabbitMqSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.coreAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getCoreArgs()); this.ruleEngineAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getRuleEngineArgs()); this.jsExecutorAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getJsExecutorArgs()); this.transportApiAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getTransportApiArgs()); this.notificationAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getNotificationsArgs()); + this.vcAdmin = new TbRabbitMqAdmin(rabbitMqSettings, queueArguments.getVcArgs()); } @Override @@ -123,6 +130,13 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(vcAdmin, rabbitMqSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbRabbitMqConsumerTemplate<>(ruleEngineAdmin, rabbitMqSettings, configuration.getTopic(), @@ -205,6 +219,11 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -222,5 +241,8 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index e728be6085..1dca3ba551 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -169,6 +170,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return builder.build(); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @Override public TbQueueConsumer> createToUsageStatsServiceMsgConsumer() { return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic(), diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..efc433c86e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments; +import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq' && '${service.type:null}'=='tb-vc-executor'") +public class RabbitMqTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbRabbitMqSettings rabbitMqSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public RabbitMqTbVersionControlQueueFactory(TbRabbitMqSettings rabbitMqSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbRabbitMqQueueArguments queueArguments + ) { + this.rabbitMqSettings = rabbitMqSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getCoreArgs()); + this.notificationAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getNotificationsArgs()); + this.vcAdmin = new TbRabbitMqAdmin(this.rabbitMqSettings, queueArguments.getVcArgs()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbRabbitMqProducerTemplate<>(notificationAdmin, rabbitMqSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(vcAdmin, rabbitMqSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index 88bb0a4045..a740a6d1a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -50,13 +51,14 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import javax.annotation.PreDestroy; import java.nio.charset.StandardCharsets; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='monolith'") -public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory { +public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngineQueueFactory, TbVersionControlQueueFactory { private final NotificationsTopicService notificationsTopicService; private final TbQueueCoreSettings coreSettings; @@ -66,12 +68,14 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul private final TbQueueTransportNotificationSettings transportNotificationSettings; private final TbServiceBusSettings serviceBusSettings; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; public ServiceBusMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -80,6 +84,7 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul TbQueueTransportNotificationSettings transportNotificationSettings, TbServiceBusSettings serviceBusSettings, TbQueueRemoteJsInvokeSettings jsInvokeSettings, + TbQueueVersionControlSettings vcSettings, TbServiceBusQueueConfigs serviceBusQueueConfigs) { this.notificationsTopicService = notificationsTopicService; this.coreSettings = coreSettings; @@ -89,12 +94,14 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul this.transportNotificationSettings = transportNotificationSettings; this.serviceBusSettings = serviceBusSettings; this.jsInvokeSettings = jsInvokeSettings; + this.vcSettings = vcSettings; this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); this.ruleEngineAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs()); this.jsExecutorAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getJsExecutorConfigs()); this.transportApiAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getTransportApiConfigs()); this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); + this.vcAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getVcConfigs()); } @Override @@ -122,6 +129,13 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, coreSettings.getTopic()); } + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(vcAdmin, serviceBusSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + @Override public TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration) { return new TbServiceBusConsumerTemplate<>(ruleEngineAdmin, serviceBusSettings, configuration.getTopic(), @@ -204,6 +218,11 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, vcSettings.getTopic()); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { @@ -221,5 +240,8 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index e1eb41b2b2..5b471b2830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; @@ -191,6 +192,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); } + @Override + public TbQueueProducer> createVersionControlMsgProducer() { + //TODO: version-control + return null; + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java new file mode 100644 index 0000000000..75e788af16 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java @@ -0,0 +1,90 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusConsumerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusProducerTemplate; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs; +import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.settings.TbQueueCoreSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; + +import javax.annotation.PreDestroy; + +@Component +@ConditionalOnExpression("'${queue.type:null}'=='service-bus' && '${service.type:null}'=='tb-vc-executor'") +public class ServiceBusTbVersionControlQueueFactory implements TbVersionControlQueueFactory { + + private final TbServiceBusSettings serviceBusSettings; + private final TbQueueCoreSettings coreSettings; + private final TbQueueVersionControlSettings vcSettings; + + private final TbQueueAdmin coreAdmin; + private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin vcAdmin; + + public ServiceBusTbVersionControlQueueFactory(TbServiceBusSettings serviceBusSettings, + TbQueueCoreSettings coreSettings, + TbQueueVersionControlSettings vcSettings, + TbServiceBusQueueConfigs serviceBusQueueConfigs + ) { + this.serviceBusSettings = serviceBusSettings; + this.coreSettings = coreSettings; + this.vcSettings = vcSettings; + + this.coreAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getCoreConfigs()); + this.notificationAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getNotificationsConfigs()); + this.vcAdmin = new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getVcConfigs()); + } + + @Override + public TbQueueProducer> createToUsageStatsServiceMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, coreSettings.getUsageStatsTopic()); + } + + @Override + public TbQueueProducer> createTbCoreNotificationsMsgProducer() { + return new TbServiceBusProducerTemplate<>(notificationAdmin, serviceBusSettings, coreSettings.getTopic()); + } + + @Override + public TbQueueConsumer> createToVersionControlMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(vcAdmin, serviceBusSettings, vcSettings.getTopic(), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToVersionControlServiceMsg.parseFrom(msg.getData()), msg.getHeaders()) + ); + } + + @PreDestroy + private void destroy() { + if (coreAdmin != null) { + coreAdmin.destroy(); + } + if (notificationAdmin != null) { + notificationAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 4debe1e280..aadefa0292 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -20,6 +20,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateSer import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -122,4 +123,11 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { TbQueueProducer> createTransportApiResponseProducer(); TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); + + /** + * Used to push messages to instances of TB Version Control Service + * + * @return + */ + TbQueueProducer> createVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index ef0b0c38e9..ed7de35274 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -22,6 +22,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -39,6 +40,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toVersionControl; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -52,6 +54,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); } @Override @@ -83,4 +86,9 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + return toVersionControl; + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 19ebb4666e..19046c5a2f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -21,6 +21,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -70,4 +71,11 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbUsageStatsMsgProducer(); + + /** + * Used to push messages to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> getTbVersionControlMsgProducer(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 26f9df069e..c21ae99b8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -83,4 +84,9 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 3132ed684f..f9cc139559 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -73,6 +74,11 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider throw new RuntimeException("Not Implemented! Should not be used by Transport!"); } + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Transport!"); + } + @Override public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java new file mode 100644 index 0000000000..c8ffc1da0d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -0,0 +1,84 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +import javax.annotation.PostConstruct; + +@Service +@ConditionalOnExpression("'${service.type:null}'=='tb-vc-executor'") +public class TbVersionControlProducerProvider implements TbQueueProducerProvider { + + private final TbVersionControlQueueFactory tbQueueProvider; + private TbQueueProducer> toTbCoreNotifications; + private TbQueueProducer> toUsageStats; + + public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { + this.tbQueueProvider = tbQueueProvider; + } + + @PostConstruct + public void init() { + this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + } + + @Override + public TbQueueProducer> getTransportNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getRuleEngineNotificationsMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbCoreNotificationsMsgProducer() { + return toTbCoreNotifications; + } + + @Override + public TbQueueProducer> getTbVersionControlMsgProducer() { + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + } + + @Override + public TbQueueProducer> getTbUsageStatsMsgProducer() { + return toUsageStats; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java new file mode 100644 index 0000000000..70b16afc68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +/** + * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. + * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable + */ +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { + + /** + * Used to push notifications to other instances of TB Core Service + * + * @return + */ + TbQueueProducer> createTbCoreNotificationsMsgProducer(); + + /** + * Used to consume messages from TB Core Service + * + * @return + */ + TbQueueConsumer> createToVersionControlMsgConsumer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java index e849d5b4f3..a4819fdad4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -37,6 +37,8 @@ public class TbPubSubSubscriptionSettings { private String notificationsProperties; @Value("${queue.pubsub.queue-properties.js-executor}") private String jsExecutorProperties; + @Value("${queue.pubsub.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreSettings; @@ -48,6 +50,8 @@ public class TbPubSubSubscriptionSettings { private Map notificationsSettings; @Getter private Map jsExecutorSettings; + @Getter + private Map vcSettings; @PostConstruct private void init() { @@ -56,6 +60,7 @@ public class TbPubSubSubscriptionSettings { transportApiSettings = getSettings(transportApiProperties); notificationsSettings = getSettings(notificationsProperties); jsExecutorSettings = getSettings(jsExecutorProperties); + vcSettings = getSettings(vcProperties); } private Map getSettings(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java index d2523f3b8f..25c78a96c5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -38,6 +38,8 @@ public class TbRabbitMqQueueArguments { private String notificationsProperties; @Value("${queue.rabbitmq.queue-properties.js-executor}") private String jsExecutorProperties; + @Value("${queue.rabbitmq.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreArgs; @@ -49,6 +51,8 @@ public class TbRabbitMqQueueArguments { private Map notificationsArgs; @Getter private Map jsExecutorArgs; + @Getter + private Map vcArgs; @PostConstruct private void init() { @@ -57,6 +61,7 @@ public class TbRabbitMqQueueArguments { transportApiArgs = getArgs(transportApiProperties); notificationsArgs = getArgs(notificationsProperties); jsExecutorArgs = getArgs(jsExecutorProperties); + vcArgs = getArgs(vcProperties); } private Map getArgs(String properties) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 33e6552eb9..f64143b14c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueCoreSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java index b930d04312..f4d62bad1d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRemoteJsInvokeSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRemoteJsInvokeSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java index eee70e9e22..57131f6b62 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueRuleEngineSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueRuleEngineSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java index 0ad9414bd8..a4dfe86bd1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportApiSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportApiSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java index d4c0064692..e2a10b31b3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueTransportNotificationSettings.java @@ -17,8 +17,10 @@ package org.thingsboard.server.queue.settings; import lombok.Data; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; +@Lazy @Data @Component public class TbQueueTransportNotificationSettings { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java new file mode 100644 index 0000000000..3a7a5602d5 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueVersionControlSettings.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.settings; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +@Lazy +@Data +@Component +public class TbQueueVersionControlSettings { + + @Value("${queue.vc.topic:tb_version_control}") + private String topic; + + @Value("${queue.vc.usage-stats-topic:tb_usage_stats}") + private String usageStatsTopic; + + @Value("${queue.vc.partitions:10}") + private int partitions; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 0cf02348f3..58704f62a7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -38,6 +38,8 @@ public class TbAwsSqsQueueAttributes { private String notificationsProperties; @Value("${queue.aws-sqs.queue-properties.js-executor}") private String jsExecutorProperties; + @Value("${queue.aws-sqs.queue-properties.version-control:}") + private String vcProperties; @Getter private Map coreAttributes; @@ -49,6 +51,8 @@ public class TbAwsSqsQueueAttributes { private Map notificationsAttributes; @Getter private Map jsExecutorAttributes; + @Getter + private Map vcAttributes; private final Map defaultAttributes = new HashMap<>(); @@ -61,6 +65,7 @@ public class TbAwsSqsQueueAttributes { transportApiAttributes = getConfigs(transportApiProperties); notificationsAttributes = getConfigs(notificationsProperties); jsExecutorAttributes = getConfigs(jsExecutorProperties); + vcAttributes = getConfigs(vcProperties); } private Map getConfigs(String properties) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java similarity index 93% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java index ed0572e8e5..8ebcc39cab 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/DataDecodingEncodingService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/DataDecodingEncodingService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import java.util.Optional; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java similarity index 96% rename from common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java rename to common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index 1c5eec383c..9d9c028951 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.transport.util; +package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java new file mode 100644 index 0000000000..e72fe60b6e --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbVersionControlComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-vc-executor'") +public @interface TbVersionControlComponent { +} diff --git a/common/queue/src/test/java/org/thingsboard/server/queue/discovery/QueueKeyTest.java b/common/queue/src/test/java/org/thingsboard/server/queue/discovery/QueueKeyTest.java new file mode 100644 index 0000000000..6fcb365f09 --- /dev/null +++ b/common/queue/src/test/java/org/thingsboard/server/queue/discovery/QueueKeyTest.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery; + +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Test; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.queue.ServiceType; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +@Slf4j +class QueueKeyTest { + + @Test + void testToStringSystemTenant() { + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "Main", TenantId.SYS_TENANT_ID); + log.info("The queue key is {}",queueKey); + assertThat(queueKey.toString()).isEqualTo("QK(Main,TB_RULE_ENGINE,system)"); + } + + @Test + void testToStringCustomTenant() { + TenantId tenantId = TenantId.fromUUID(UUID.fromString("3ebd39eb-43d4-4911-a818-cdbf8d508f88")); + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "Main", tenantId); + log.info("The queue key is {}",queueKey); + assertThat(queueKey.toString()).isEqualTo("QK(Main,TB_RULE_ENGINE,3ebd39eb-43d4-4911-a818-cdbf8d508f88)"); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index c75de25267..a10c3e4969 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -24,14 +24,11 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @TbSnmpTransportComponent @Service diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 8b3f9feb22..51373e2a4a 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -64,10 +64,6 @@ com.google.code.gson gson - - de.ruedigermoeller - fst - org.slf4j slf4j-api diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index b79306199f..2b0f73343d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java index 666941b143..b4187432e8 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportResourceCache.java @@ -24,7 +24,7 @@ import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.transport.TransportResourceCache; import org.thingsboard.server.common.transport.TransportService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 9f6ddc87a4..68f8db272b 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -69,7 +69,7 @@ import org.thingsboard.server.common.transport.auth.GetOrCreateDeviceFromGateway import org.thingsboard.server.common.transport.auth.TransportDeviceInfo; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; @@ -901,7 +901,7 @@ public class DefaultTransportService implements TransportService { if (EntityType.DEVICE_PROFILE.equals(entityType)) { DeviceProfile deviceProfile = deviceProfileCache.put(msg.getData()); if (deviceProfile != null) { - log.info("On device profile update: {}", deviceProfile); + log.debug("On device profile update: {}", deviceProfile); onProfileUpdate(deviceProfile); } } else if (EntityType.TENANT_PROFILE.equals(entityType)) { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index 4cfefb7a6f..bb7f8f3f4d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportTenantProfileCache; import org.thingsboard.server.common.transport.limits.TransportRateLimitService; import org.thingsboard.server.common.transport.profile.TenantProfileUpdateResult; -import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index f9df92966b..9c4b68d3c0 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -18,13 +18,20 @@ package org.thingsboard.common.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.UUID; +import java.util.function.UnaryOperator; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -32,6 +39,11 @@ import java.util.Set; public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final ObjectMapper PRETTY_SORTED_JSON_MAPPER = JsonMapper.builder() + .enable(SerializationFeature.INDENT_OUTPUT) + .configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true) + .configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true) + .build(); public static T convertValue(Object fromValue, Class toValueType) { try { @@ -96,6 +108,14 @@ public class JacksonUtil { } } + public static String toPrettyString(Object o) { + try { + return PRETTY_SORTED_JSON_MAPPER.writeValueAsString(o); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + public static JsonNode toJsonNode(String value) { if (value == null || value.isEmpty()) { return null; @@ -137,4 +157,59 @@ public class JacksonUtil { + value + " cannot be transformed to a String", e); } } + + + public static JsonNode getSafely(JsonNode node, String... path) { + if (node == null) { + return null; + } + for (String p : path) { + if (!node.has(p)) { + return null; + } else { + node = node.get(p); + } + } + return node; + } + + public static void replaceUuidsRecursively(JsonNode node, Set skipFieldsSet, UnaryOperator replacer) { + if (node == null) { + return; + } + if (node.isObject()) { + ObjectNode objectNode = (ObjectNode) node; + List fieldNames = new ArrayList<>(objectNode.size()); + objectNode.fieldNames().forEachRemaining(fieldNames::add); + for (String fieldName : fieldNames) { + if (skipFieldsSet.contains(fieldName)) { + continue; + } + var child = objectNode.get(fieldName); + if (child.isObject() || child.isArray()) { + replaceUuidsRecursively(child, skipFieldsSet, replacer); + } else if (child.isTextual()) { + String text = child.asText(); + String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); + if (!text.equals(newText)) { + objectNode.put(fieldName, newText); + } + } + } + } else if (node.isArray()) { + ArrayNode array = (ArrayNode) node; + for (int i = 0; i < array.size(); i++) { + JsonNode arrayElement = array.get(i); + if (arrayElement.isObject() || arrayElement.isArray()) { + replaceUuidsRecursively(arrayElement, skipFieldsSet, replacer); + } else if (arrayElement.isTextual()) { + String text = arrayElement.asText(); + String newText = RegexUtils.replace(text, RegexUtils.UUID_PATTERN, uuid -> replacer.apply(UUID.fromString(uuid)).toString()); + if (!text.equals(newText)) { + array.set(i, newText); + } + } + } + } + } } diff --git a/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java b/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java new file mode 100644 index 0000000000..af968a6bff --- /dev/null +++ b/common/util/src/main/java/org/thingsboard/common/util/RegexUtils.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.common.util; + +import lombok.AccessLevel; +import lombok.NoArgsConstructor; + +import java.util.function.UnaryOperator; +import java.util.regex.Pattern; + +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class RegexUtils { + + public static final Pattern UUID_PATTERN = Pattern.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"); + + + public static String replace(String s, Pattern pattern, UnaryOperator replacer) { + return pattern.matcher(s).replaceAll(matchResult -> { + return replacer.apply(matchResult.group()); + }); + } + +} diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java index 13a0d466ac..d8b4cbf720 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java @@ -28,12 +28,23 @@ import org.springframework.util.StopWatch; * */ public class TbStopWatch extends StopWatch { - public static TbStopWatch startNew(){ + public static TbStopWatch create(){ TbStopWatch stopWatch = new TbStopWatch(); stopWatch.start(); return stopWatch; } + public static TbStopWatch create(String taskName){ + TbStopWatch stopWatch = new TbStopWatch(); + stopWatch.start(taskName); + return stopWatch; + } + + public void startNew(String taskName){ + stop(); + start(taskName); + } + public long stopAndGetTotalTimeMillis(){ stop(); return getTotalTimeMillis(); diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml new file mode 100644 index 0000000000..998ae0879b --- /dev/null +++ b/common/version-control/pom.xml @@ -0,0 +1,125 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + common + + org.thingsboard.common + version-control + jar + + Thingsboard Server Version Control API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.thingsboard.common + data + + + org.thingsboard.common + queue + + + org.springframework + spring-core + + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + + + com.google.guava + guava + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.eclipse.jgit + org.eclipse.jgit + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + + thingsboard-repo-deploy + ThingsBoard Repo Deployment + https://repo.thingsboard.io/artifactory/libs-release-public + + + + diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java new file mode 100644 index 0000000000..0823dd06e6 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/ClusterVersionControlService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; + +public interface ClusterVersionControlService extends ApplicationListener { +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java new file mode 100644 index 0000000000..f74f18ddaa --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -0,0 +1,546 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.AddMsg; +import org.thingsboard.server.gen.transport.TransportProtos.BranchInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityContentResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.EntityVersionProto; +import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListBranchesResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.VersionedEntityInfoProto; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.discovery.NotificationsTopicService; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; +import org.thingsboard.server.queue.provider.TbVersionControlQueueFactory; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.queue.util.TbVersionControlComponent; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.thingsboard.server.service.sync.vc.DefaultGitRepositoryService.fromRelativePath; + +@Slf4j +@TbVersionControlComponent +@Service +@RequiredArgsConstructor +public class DefaultClusterVersionControlService extends TbApplicationEventListener implements ClusterVersionControlService { + + private final PartitionService partitionService; + private final TbQueueProducerProvider producerProvider; + private final TbVersionControlQueueFactory queueFactory; + private final DataDecodingEncodingService encodingService; + private final GitRepositoryService vcService; + private final NotificationsTopicService notificationsTopicService; + + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + private volatile ExecutorService consumerExecutor; + private volatile TbQueueConsumer> consumer; + private volatile TbQueueProducer> producer; + private volatile boolean stopped = false; + + @Value("${queue.vc.poll-interval:25}") + private long pollDuration; + @Value("${queue.vc.pack-processing-timeout:60000}") + private long packProcessingTimeout; + @Value("${vc.git.io_pool_size:3}") + private int ioPoolSize; + + //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. + private final List ioThreads = new ArrayList<>(); + + + @PostConstruct + public void init() { + consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("vc-consumer")); + var threadFactory = ThingsBoardThreadFactory.forName("vc-io-thread"); + for (int i = 0; i < ioPoolSize; i++) { + ioThreads.add(MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(threadFactory))); + } + producer = producerProvider.getTbCoreNotificationsMsgProducer(); + consumer = queueFactory.createToVersionControlMsgConsumer(); + } + + @PreDestroy + public void stop() { + stopped = true; + if (consumer != null) { + consumer.unsubscribe(); + } + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + ioThreads.forEach(ExecutorService::shutdownNow); + } + + @Override + protected void onTbApplicationEvent(PartitionChangeEvent event) { + for (TenantId tenantId : vcService.getActiveRepositoryTenants()) { + if (!partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId).isMyPartition()) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + pendingCommitMap.remove(tenantId); + vcService.clearRepository(tenantId); + } catch (Exception e) { + log.warn("[{}] Failed to cleanup the tenant repository", tenantId, e); + } finally { + lock.unlock(); + } + } + } + consumer.subscribe(event.getPartitions()); + } + + @Override + protected boolean filterTbApplicationEvent(PartitionChangeEvent event) { + return ServiceType.TB_VC_EXECUTOR.equals(event.getServiceType()); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent event) { + consumerExecutor.execute(() -> consumerLoop(consumer)); + } + + void consumerLoop(TbQueueConsumer> consumer) { + while (!stopped && !consumer.isStopped()) { + List> futures = new ArrayList<>(); + try { + List> msgs = consumer.poll(pollDuration); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msgWrapper : msgs) { + ToVersionControlServiceMsg msg = msgWrapper.getValue(); + var ctx = new VersionControlRequestCtx(msg, msg.hasClearRepositoryRequest() ? null : getEntitiesVersionControlSettings(msg)); + long startTs = System.currentTimeMillis(); + log.trace("[{}][{}] RECEIVED task: {}", ctx.getTenantId(), ctx.getRequestId(), msg); + int threadIdx = Math.abs(ctx.getTenantId().hashCode() % ioPoolSize); + ListenableFuture future = ioThreads.get(threadIdx).submit(() -> processMessage(ctx, msg)); + logTaskExecution(ctx, future, startTs); + futures.add(future); + } + try { + Futures.allAsList(futures).get(packProcessingTimeout, TimeUnit.MILLISECONDS); + } catch (TimeoutException e) { + log.info("Timeout for processing the version control tasks.", e); + } + consumer.commit(); + } catch (Exception e) { + if (!stopped) { + log.warn("Failed to obtain version control requests from queue.", e); + try { + Thread.sleep(pollDuration); + } catch (InterruptedException e2) { + log.trace("Failed to wait until the server has capacity to handle new version control messages", e2); + } + } + } + } + log.info("TB Version Control request consumer stopped."); + } + + private Void processMessage(VersionControlRequestCtx ctx, ToVersionControlServiceMsg msg) { + var lock = getRepoLock(ctx.getTenantId()); + lock.lock(); + try { + if (msg.hasClearRepositoryRequest()) { + handleClearRepositoryCommand(ctx); + } else { + if (msg.hasTestRepositoryRequest()) { + handleTestRepositoryCommand(ctx); + } else if (msg.hasInitRepositoryRequest()) { + handleInitRepositoryCommand(ctx); + } else { + var currentSettings = vcService.getRepositorySettings(ctx.getTenantId()); + var newSettings = ctx.getSettings(); + if (!newSettings.equals(currentSettings)) { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + } + if (msg.hasCommitRequest()) { + handleCommitRequest(ctx, msg.getCommitRequest()); + } else if (msg.hasListBranchesRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListBranches(ctx, msg.getListBranchesRequest()); + } else if (msg.hasListEntitiesRequest()) { + handleListEntities(ctx, msg.getListEntitiesRequest()); + } else if (msg.hasListVersionRequest()) { + vcService.fetch(ctx.getTenantId()); + handleListVersions(ctx, msg.getListVersionRequest()); + } else if (msg.hasEntityContentRequest()) { + handleEntityContentRequest(ctx, msg.getEntityContentRequest()); + } else if (msg.hasEntitiesContentRequest()) { + handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); + } else if (msg.hasVersionsDiffRequest()) { + handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); + } else if (msg.hasContentsDiffRequest()) { + handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); + } + } + } + } catch (Exception e) { + reply(ctx, Optional.of(e)); + } finally { + lock.unlock(); + } + return null; + } + + private void handleEntitiesContentRequest(VersionControlRequestCtx ctx, EntitiesContentRequestMsg request) throws Exception { + var entityType = EntityType.valueOf(request.getEntityType()); + String path = getRelativePath(entityType, null); + var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) + .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); + var response = EntitiesContentResponseMsg.newBuilder(); + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + response.addData(data); + } + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + } + + private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { + String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + } + + private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { + String path; + if (StringUtils.isNotEmpty(request.getEntityType())) { + var entityType = EntityType.valueOf(request.getEntityType()); + if (request.getEntityIdLSB() != 0 || request.getEntityIdMSB() != 0) { + path = getRelativePath(entityType, new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); + } else { + path = getRelativePath(entityType, null); + } + } else { + path = null; + } + SortOrder sortOrder = null; + if (StringUtils.isNotEmpty(request.getSortProperty())) { + var direction = SortOrder.Direction.DESC; + if (StringUtils.isNotEmpty(request.getSortDirection())) { + direction = SortOrder.Direction.valueOf(request.getSortDirection()); + } + sortOrder = new SortOrder(request.getSortProperty(), direction); + } + var data = vcService.listVersions(ctx.getTenantId(), request.getBranchName(), path, + new PageLink(request.getPageSize(), request.getPage(), request.getTextSearch(), sortOrder)); + reply(ctx, Optional.empty(), builder -> + builder.setListVersionsResponse(ListVersionsResponseMsg.newBuilder() + .setTotalPages(data.getTotalPages()) + .setTotalElements(data.getTotalElements()) + .setHasNext(data.hasNext()) + .addAllVersions(data.getData().stream().map( + v -> EntityVersionProto.newBuilder().setTs(v.getTimestamp()).setId(v.getId()).setName(v.getName()).setAuthor(v.getAuthor()).build() + ).collect(Collectors.toList()))) + ); + } + + private void handleListEntities(VersionControlRequestCtx ctx, ListEntitiesRequestMsg request) throws Exception { + EntityType entityType = StringUtils.isNotEmpty(request.getEntityType()) ? EntityType.valueOf(request.getEntityType()) : null; + var path = entityType != null ? getRelativePath(entityType, null) : null; + var data = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path); + reply(ctx, Optional.empty(), builder -> + builder.setListEntitiesResponse(ListEntitiesResponseMsg.newBuilder() + .addAllEntities(data.stream().map(VersionedEntityInfo::getExternalId).map( + id -> VersionedEntityInfoProto.newBuilder() + .setEntityType(id.getEntityType().name()) + .setEntityIdMSB(id.getId().getMostSignificantBits()) + .setEntityIdLSB(id.getId().getLeastSignificantBits()).build() + ).collect(Collectors.toList())))); + } + + private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { + var branches = vcService.listBranches(ctx.getTenantId()).stream() + .map(branchInfo -> BranchInfoProto.newBuilder() + .setName(branchInfo.getName()) + .setIsDefault(branchInfo.isDefault()).build()).collect(Collectors.toList()); + reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); + } + + private void handleVersionsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.VersionsDiffRequestMsg request) throws IOException { + List diffList = vcService.getVersionsDiffList(ctx.getTenantId(), request.getPath(), request.getVersionId1(), request.getVersionId2()).stream() + .map(diff -> { + EntityId entityId = fromRelativePath(diff.getFilePath()); + return TransportProtos.EntityVersionsDiff.newBuilder() + .setEntityType(entityId.getEntityType().name()) + .setEntityIdMSB(entityId.getId().getMostSignificantBits()) + .setEntityIdLSB(entityId.getId().getLeastSignificantBits()) + .setEntityDataAtVersion1(diff.getFileContentAtCommit1()) + .setEntityDataAtVersion2(diff.getFileContentAtCommit2()) + .setRawDiff(diff.getDiffStringValue()) + .build(); + }) + .collect(Collectors.toList()); + + reply(ctx, builder -> builder.setVersionsDiffResponse(TransportProtos.VersionsDiffResponseMsg.newBuilder() + .addAllDiff(diffList))); + } + + private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { + String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); + reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() + .setDiff(diff))); + } + + private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { + var tenantId = ctx.getTenantId(); + UUID txId = UUID.fromString(request.getTxId()); + if (request.hasPrepareMsg()) { + vcService.fetch(ctx.getTenantId()); + prepareCommit(ctx, txId, request.getPrepareMsg()); + } else if (request.hasAbortMsg()) { + PendingCommit current = pendingCommitMap.get(tenantId); + if (current != null && current.getTxId().equals(txId)) { + doAbortCurrentCommit(tenantId, current); + } + } else { + PendingCommit current = pendingCommitMap.get(tenantId); + if (current != null && current.getTxId().equals(txId)) { + try { + if (request.hasAddMsg()) { + addToCommit(ctx, current, request.getAddMsg()); + } else if (request.hasDeleteMsg()) { + deleteFromCommit(ctx, current, request.getDeleteMsg()); + } else if (request.hasPushMsg()) { + var result = vcService.push(current); + pendingCommitMap.remove(ctx.getTenantId()); + reply(ctx, result); + } + } catch (Exception e) { + doAbortCurrentCommit(tenantId, current, e); + throw e; + } + } else { + log.debug("[{}] Ignore request due to stale commit: {}", txId, request); + } + } + } + + private void prepareCommit(VersionControlRequestCtx ctx, UUID txId, PrepareMsg prepareMsg) { + var tenantId = ctx.getTenantId(); + var pendingCommit = new PendingCommit(tenantId, ctx.getNodeId(), txId, prepareMsg.getBranchName(), + prepareMsg.getCommitMsg(), prepareMsg.getAuthorName(), prepareMsg.getAuthorEmail()); + PendingCommit old = pendingCommitMap.get(tenantId); + if (old != null) { + doAbortCurrentCommit(tenantId, old); + } + pendingCommitMap.put(tenantId, pendingCommit); + vcService.prepareCommit(pendingCommit); + } + + private void deleteFromCommit(VersionControlRequestCtx ctx, PendingCommit commit, DeleteMsg deleteMsg) throws IOException { + vcService.deleteFolderContent(commit, deleteMsg.getRelativePath()); + } + + private void addToCommit(VersionControlRequestCtx ctx, PendingCommit commit, AddMsg addMsg) throws IOException { + vcService.add(commit, addMsg.getRelativePath(), addMsg.getEntityDataJson()); + } + + private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current) { + doAbortCurrentCommit(tenantId, current, null); + } + + private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current, Exception e) { + vcService.abort(current); + pendingCommitMap.remove(tenantId); + //TODO: push notification to core using old.getNodeId() to cancel old commit processing on the caller side. + } + + private void handleClearRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.clearRepository(ctx.getTenantId()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void handleInitRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.initRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + + private void handleTestRepositoryCommand(VersionControlRequestCtx ctx) { + try { + vcService.testRepository(ctx.getTenantId(), ctx.getSettings()); + reply(ctx, Optional.empty()); + } catch (Exception e) { + log.debug("[{}] Failed to connect to the repository: ", ctx, e); + reply(ctx, Optional.of(e)); + } + } + + private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) { + var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded()) + .setModified(result.getModified()) + .setRemoved(result.getRemoved()); + + if (result.getVersion() != null) { + responseBuilder.setTs(result.getVersion().getTimestamp()) + .setCommitId(result.getVersion().getId()) + .setName(result.getVersion().getName()) + .setAuthor(result.getVersion().getAuthor()); + } + + reply(ctx, Optional.empty(), builder -> builder.setCommitResponse(responseBuilder)); + } + + private void reply(VersionControlRequestCtx ctx, Optional e) { + reply(ctx, e, null); + } + + private void reply(VersionControlRequestCtx ctx, Function enrichFunction) { + reply(ctx, Optional.empty(), enrichFunction); + } + + private void reply(VersionControlRequestCtx ctx, Optional e, Function enrichFunction) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, ctx.getNodeId()); + VersionControlResponseMsg.Builder builder = VersionControlResponseMsg.newBuilder() + .setRequestIdMSB(ctx.getRequestId().getMostSignificantBits()) + .setRequestIdLSB(ctx.getRequestId().getLeastSignificantBits()); + if (e.isPresent()) { + log.debug("[{}][{}] Failed to process task", ctx.getTenantId(), ctx.getRequestId(), e.get()); + builder.setError(e.get().getMessage()); + } else { + if (enrichFunction != null) { + builder = enrichFunction.apply(builder); + } else { + builder.setGenericResponse(TransportProtos.GenericRepositoryResponseMsg.newBuilder().build()); + } + log.debug("[{}][{}] Processed task", ctx.getTenantId(), ctx.getRequestId()); + } + + ToCoreNotificationMsg msg = ToCoreNotificationMsg.newBuilder().setVcResponseMsg(builder).build(); + log.trace("[{}][{}] PUSHING reply: {} to: {}", ctx.getTenantId(), ctx.getRequestId(), msg, tpi); + producer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + } + + private RepositorySettings getEntitiesVersionControlSettings(ToVersionControlServiceMsg msg) { + Optional settingsOpt = encodingService.decode(msg.getVcSettings().toByteArray()); + if (settingsOpt.isPresent()) { + return settingsOpt.get(); + } else { + log.warn("Failed to parse VC settings: {}", msg.getVcSettings()); + throw new RuntimeException("Failed to parse vc settings!"); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock(true)); + } + + private void logTaskExecution(VersionControlRequestCtx ctx, ListenableFuture future, long startTs) { + if (log.isTraceEnabled()) { + Futures.addCallback(future, new FutureCallback() { + + @Override + public void onSuccess(@Nullable Object result) { + log.trace("[{}][{}] Task processing took: {}ms", ctx.getTenantId(), ctx.getRequestId(), (System.currentTimeMillis() - startTs)); + } + + @Override + public void onFailure(Throwable t) { + log.trace("[{}][{}] Task failed: ", ctx.getTenantId(), ctx.getRequestId(), t); + } + }, MoreExecutors.directExecutor()); + } + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..d3615f7228 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,276 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; + +import javax.annotation.PostConstruct; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${java.io.tmpdir}/repositories") + private String defaultFolder; + + @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + if (StringUtils.isEmpty(repositoriesFolder)) { + repositoriesFolder = defaultFolder; + } + } + + @Override + public Set getActiveRepositoryTenants() { + return new HashSet<>(repositories.keySet()); + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getBranch(); + try { + repository.fetch(); + + repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); + repository.resetAndClean(); + + if (repository.listRemoteBranches().contains(new BranchInfo(branch, false))) { + repository.merge(branch); + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + if (result.getAdded() > 0 || result.getModified() > 0 || result.getRemoved() > 0) { + GitRepository.Commit gitCommit = repository.commit(commit.getVersionName(), commit.getAuthorName(), commit.getAuthorEmail()); + repository.push(commit.getWorkingBranch(), commit.getBranch()); + result.setVersion(toVersion(gitCommit)); + } + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } finally { + cleanUp(commit); + } + } + + @SneakyThrows + @Override + public void cleanUp(PendingCommit commit) { + log.debug("[{}] Cleanup tenant repository started.", commit.getTenantId()); + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString()); + } catch (Exception e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + repository.resetAndClean(); + repository.deleteLocalBranchIfExists(commit.getWorkingBranch()); + log.debug("[{}] Cleanup tenant repository completed.", commit.getTenantId()); + } + + @Override + public void abort(PendingCommit commit) { + cleanUp(commit); + } + + @Override + public void fetch(TenantId tenantId) throws GitAPIException { + var repository = repositories.get(tenantId); + if (repository != null) { + log.debug("[{}] Fetching tenant repository.", tenantId); + repository.fetch(); + log.debug("[{}] Fetched tenant repository.", tenantId); + } + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getDiffList(versionId1, versionId2, path); + } + + @Override + public String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getContentsDiff(content1, content2); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listRemoteBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, pageLink).mapData(this::toVersion); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository.test(settings, repositoryDirectory.toFile()); + } + + @Override + public void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception { + clearRepository(tenantId); + log.debug("[{}] Init tenant repository started.", tenantId); + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings, repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + log.debug("[{}] Init tenant repository completed.", tenantId); + } + + @Override + public RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception { + var gitRepository = repositories.get(tenantId); + return gitRepository != null ? gitRepository.getSettings() : null; + } + + @Override + public void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + log.debug("[{}] Clear tenant repository started.", tenantId); + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + log.debug("[{}] Clear tenant repository completed.", tenantId); + } + } + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getTimestamp(), commit.getId(), commit.getMessage(), this.getAuthor(commit)); + } + + private String getAuthor(GitRepository.Commit commit) { + String author = String.format("<%s>", commit.getAuthorEmail()); + if (StringUtils.isNotBlank(commit.getAuthorName())) { + author = String.format("%s %s", commit.getAuthorName(), author); + } + return author; + } + + public static EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java new file mode 100644 index 0000000000..392c686a14 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -0,0 +1,513 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Ordering; +import com.google.common.collect.Streams; +import lombok.Data; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.apache.sshd.common.util.security.SecurityUtils; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.GitCommand; +import org.eclipse.jgit.api.ListBranchCommand; +import org.eclipse.jgit.api.LogCommand; +import org.eclipse.jgit.api.LsRemoteCommand; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.TransportCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.diff.DiffFormatter; +import org.eclipse.jgit.diff.EditList; +import org.eclipse.jgit.diff.HistogramDiff; +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.diff.RawTextComparator; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.filter.RevFilter; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.FetchResult; +import org.eclipse.jgit.transport.RefSpec; +import org.eclipse.jgit.transport.SshTransport; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.transport.sshd.JGitKeyCache; +import org.eclipse.jgit.transport.sshd.ServerKeyDatabase; +import org.eclipse.jgit.transport.sshd.SshdSessionFactory; +import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.security.KeyPair; +import java.security.PublicKey; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class GitRepository { + + private final Git git; + @Getter + private final RepositorySettings settings; + private final CredentialsProvider credentialsProvider; + private final SshdSessionFactory sshSessionFactory; + + @Getter + private final String directory; + + private ObjectId headId; + + private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { + this.git = git; + this.settings = settings; + this.credentialsProvider = credentialsProvider; + this.sshSessionFactory = sshSessionFactory; + this.directory = directory; + } + + public static GitRepository clone(RepositorySettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + CloneCommand cloneCommand = Git.cloneRepository() + .setURI(settings.getRepositoryUri()) + .setDirectory(directory) + .setNoCheckout(true); + configureTransportCommand(cloneCommand, credentialsProvider, sshSessionFactory); + Git git = cloneCommand.call(); + return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); + } + + public static GitRepository open(File directory, RepositorySettings settings) throws IOException { + Git git = Git.open(directory); + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + return new GitRepository(git, settings, credentialsProvider, sshSessionFactory, directory.getAbsolutePath()); + } + + public static void test(RepositorySettings settings, File directory) throws GitAPIException { + CredentialsProvider credentialsProvider = null; + SshdSessionFactory sshSessionFactory = null; + if (RepositoryAuthMethod.USERNAME_PASSWORD.equals(settings.getAuthMethod())) { + credentialsProvider = newCredentialsProvider(settings.getUsername(), settings.getPassword()); + } else if (RepositoryAuthMethod.PRIVATE_KEY.equals(settings.getAuthMethod())) { + sshSessionFactory = newSshdSessionFactory(settings.getPrivateKey(), settings.getPrivateKeyPassword(), directory); + } + LsRemoteCommand lsRemoteCommand = Git.lsRemoteRepository().setRemote(settings.getRepositoryUri()); + configureTransportCommand(lsRemoteCommand, credentialsProvider, sshSessionFactory); + lsRemoteCommand.call(); + } + + public void fetch() throws GitAPIException { + FetchResult result = execute(git.fetch() + .setRemoveDeletedRefs(true)); + Ref head = result.getAdvertisedRef(Constants.HEAD); + if (head != null) { + this.headId = head.getObjectId(); + } + } + + public void deleteLocalBranchIfExists(String branch) throws GitAPIException { + execute(git.branchDelete() + .setBranchNames(branch) + .setForce(true)); + } + + public void resetAndClean() throws GitAPIException { + execute(git.reset() + .setMode(ResetCommand.ResetType.HARD)); + execute(git.clean() + .setForce(true) + .setCleanDirectories(true)); + } + + public void merge(String branch) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + + public List listRemoteBranches() throws GitAPIException { + return execute(git.branchList() + .setListMode(ListBranchCommand.ListMode.REMOTE)).stream() + .filter(ref -> !ref.getName().equals(Constants.HEAD)) + .map(this::toBranchInfo) + .distinct().collect(Collectors.toList()); + } + + public PageData listCommits(String branch, PageLink pageLink) throws IOException, GitAPIException { + return listCommits(branch, null, pageLink); + } + + public PageData listCommits(String branch, String path, PageLink pageLink) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + return new PageData<>(); + } + LogCommand command = git.log() + .add(branchId); + + if (StringUtils.isNotEmpty(pageLink.getTextSearch())) { + command.setRevFilter(new NoMergesAndCommitMessageFilter(pageLink.getTextSearch())); + } else { + command.setRevFilter(RevFilter.NO_MERGES); + } + + if (StringUtils.isNotEmpty(path)) { + command.addPath(path); + } + + Iterable commits = execute(command); + return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction); + } + + public List listFilesAtCommit(String commitId) throws IOException { + return listFilesAtCommit(commitId, null); + } + + public List listFilesAtCommit(String commitId, String path) throws IOException { + List files = new ArrayList<>(); + RevCommit revCommit = resolveCommit(commitId); + try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { + treeWalk.reset(revCommit.getTree().getId()); + if (StringUtils.isNotEmpty(path)) { + treeWalk.setFilter(PathFilter.create(path)); + } + treeWalk.setRecursive(true); + while (treeWalk.next()) { + files.add(treeWalk.getPathString()); + } + } + return files; + } + + + public String getFileContentAtCommit(String file, String commitId) throws IOException { + RevCommit revCommit = resolveCommit(commitId); + try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), file, revCommit.getTree())) { + if (treeWalk == null) { + throw new IllegalArgumentException("File not found"); + } + ObjectId blobId = treeWalk.getObjectId(0); + try (ObjectReader objectReader = git.getRepository().newObjectReader()) { + ObjectLoader objectLoader = objectReader.open(blobId); + byte[] bytes = objectLoader.getBytes(); + return new String(bytes, StandardCharsets.UTF_8); + } + } + } + + + public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { + execute(git.checkout() + .setOrphan(true) + .setForced(true) + .setName(name)); + } + + public void add(String filesPattern) throws GitAPIException { + execute(git.add().setUpdate(true).addFilepattern(filesPattern)); + execute(git.add().addFilepattern(filesPattern)); + } + + public Status status() throws GitAPIException { + org.eclipse.jgit.api.Status status = execute(git.status()); + Set modified = new HashSet<>(); + modified.addAll(status.getModified()); + modified.addAll(status.getChanged()); + return new Status(status.getAdded(), modified, status.getRemoved()); + } + + public Commit commit(String message, String authorName, String authorEmail) throws GitAPIException { + RevCommit revCommit = execute(git.commit() + .setAuthor(authorName, authorEmail) + .setMessage(message)); + return toCommit(revCommit); + } + + + public void push(String localBranch, String remoteBranch) throws GitAPIException { + execute(git.push() + .setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch))); + } + + public String getContentsDiff(String content1, String content2) throws IOException { + RawText rawContent1 = new RawText(content1.getBytes()); + RawText rawContent2 = new RawText(content2.getBytes()); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + + EditList edits = new EditList(); + edits.addAll(new HistogramDiff().diff(RawTextComparator.DEFAULT, rawContent1, rawContent2)); + diffFormatter.format(edits, rawContent1, rawContent2); + return out.toString(); + } + + public List getDiffList(String commit1, String commit2, String path) throws IOException { + ObjectReader reader = git.getRepository().newObjectReader(); + + CanonicalTreeParser tree1Iter = new CanonicalTreeParser(); + ObjectId tree1 = resolveCommit(commit1).getTree(); + tree1Iter.reset(reader, tree1); + + CanonicalTreeParser tree2Iter = new CanonicalTreeParser(); + ObjectId tree2 = resolveCommit(commit2).getTree(); + tree2Iter.reset(reader, tree2); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(out); + diffFormatter.setRepository(git.getRepository()); + if (StringUtils.isNotEmpty(path)) { + diffFormatter.setPathFilter(PathFilter.create(path)); + } + + return diffFormatter.scan(tree1, tree2).stream() + .map(diffEntry -> { + Diff diff = new Diff(); + try { + out.reset(); + diffFormatter.format(diffEntry); + diff.setDiffStringValue(out.toString()); + diff.setFilePath(diffEntry.getChangeType() != DiffEntry.ChangeType.DELETE ? diffEntry.getNewPath() : diffEntry.getOldPath()); + diff.setChangeType(diffEntry.getChangeType()); + try { + diff.setFileContentAtCommit1(getFileContentAtCommit(diff.getFilePath(), commit1)); + } catch (IllegalArgumentException ignored) { + } + try { + diff.setFileContentAtCommit2(getFileContentAtCommit(diff.getFilePath(), commit2)); + } catch (IllegalArgumentException ignored) { + } + return diff; + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toList()); + } + + private BranchInfo toBranchInfo(Ref ref) { + String name = org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()); + String branchName = StringUtils.removeStart(name, "origin/"); + boolean isDefault = this.headId != null && this.headId.equals(ref.getObjectId()); + return new BranchInfo(branchName, isDefault); + } + + private Commit toCommit(RevCommit revCommit) { + return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), + revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); + } + + private RevCommit resolveCommit(String id) throws IOException { + return git.getRepository().parseCommit(resolve(id)); + } + + private ObjectId resolve(String rev) throws IOException { + return git.getRepository().resolve(rev); + } + + private , T> T execute(C command) throws GitAPIException { + if (command instanceof TransportCommand) { + configureTransportCommand((TransportCommand) command, credentialsProvider, sshSessionFactory); + } + return command.call(); + } + + private static Function> revCommitComparatorFunction = pageLink -> { + SortOrder sortOrder = pageLink.getSortOrder(); + if (sortOrder != null + && sortOrder.getProperty().equals("timestamp") + && SortOrder.Direction.ASC.equals(sortOrder.getDirection())) { + return Comparator.comparingInt(RevCommit::getCommitTime); + } + return null; + }; + + private static PageData iterableToPageData(Iterable iterable, + Function mapper, + PageLink pageLink, + Function> comparatorFunction) { + iterable = Streams.stream(iterable).collect(Collectors.toList()); + int totalElements = Iterables.size(iterable); + int totalPages = pageLink.getPageSize() > 0 ? (int) Math.ceil((float) totalElements / pageLink.getPageSize()) : 1; + int startIndex = pageLink.getPageSize() * pageLink.getPage(); + int limit = startIndex + pageLink.getPageSize(); + if (comparatorFunction != null) { + Comparator comparator = comparatorFunction.apply(pageLink); + if (comparator != null) { + iterable = Ordering.from(comparator).immutableSortedCopy(iterable); + } + } + iterable = Iterables.limit(iterable, limit); + if (startIndex < totalElements) { + iterable = Iterables.skip(iterable, startIndex); + } else { + iterable = Collections.emptyList(); + } + List data = Streams.stream(iterable).map(mapper) + .collect(Collectors.toList()); + boolean hasNext = pageLink.getPageSize() > 0 && totalElements > startIndex + data.size(); + return new PageData<>(data, totalPages, totalElements, hasNext); + } + + private static void configureTransportCommand(TransportCommand transportCommand, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory) { + if (credentialsProvider != null) { + transportCommand.setCredentialsProvider(credentialsProvider); + } + if (sshSessionFactory != null) { + transportCommand.setTransportConfigCallback(transport -> { + if (transport instanceof SshTransport) { + SshTransport sshTransport = (SshTransport) transport; + sshTransport.setSshSessionFactory(sshSessionFactory); + } + }); + } + } + + private static CredentialsProvider newCredentialsProvider(String username, String password) { + return new UsernamePasswordCredentialsProvider(username, password == null ? "" : password); + } + + private static SshdSessionFactory newSshdSessionFactory(String privateKey, String password, File directory) { + SshdSessionFactory sshSessionFactory = null; + if (StringUtils.isNotBlank(privateKey)) { + Iterable keyPairs = loadKeyPairs(privateKey, password); + sshSessionFactory = new SshdSessionFactoryBuilder() + .setPreferredAuthentications("publickey") + .setDefaultKeysProvider(file -> keyPairs) + .setHomeDirectory(directory) + .setSshDirectory(directory) + .setServerKeyDatabase((file, file2) -> new ServerKeyDatabase() { + @Override + public List lookup(String connectAddress, InetSocketAddress remoteAddress, Configuration config) { + return Collections.emptyList(); + } + + @Override + public boolean accept(String connectAddress, InetSocketAddress remoteAddress, PublicKey serverKey, Configuration config, CredentialsProvider provider) { + return true; + } + }) + .build(new JGitKeyCache()); + } + return sshSessionFactory; + } + + private static Iterable loadKeyPairs(String privateKeyContent, String password) { + Iterable keyPairs = null; + try { + keyPairs = SecurityUtils.loadKeyPairIdentities(null, + null, new ByteArrayInputStream(privateKeyContent.getBytes()), (session, resourceKey, retryIndex) -> password); + } catch (Exception e) {} + if (keyPairs == null) { + throw new IllegalArgumentException("Failed to load ssh private key"); + } + return keyPairs; + } + + private static class NoMergesAndCommitMessageFilter extends RevFilter { + + private final String textSearch; + + NoMergesAndCommitMessageFilter(String textSearch) { + this.textSearch = textSearch.toLowerCase(); + } + + @Override + public boolean include(RevWalk walker, RevCommit c) { + return c.getParentCount() < 2 && c.getFullMessage().toLowerCase().contains(this.textSearch); + } + + @Override + public RevFilter clone() { + return this; + } + + @Override + public boolean requiresCommitBody() { + return false; + } + + @Override + public String toString() { + return "NO_MERGES_AND_COMMIT_MESSAGE"; + } + } + + @Data + public static class Commit { + private final long timestamp; + private final String id; + private final String message; + private final String authorName; + private final String authorEmail; + } + + @Data + public static class Status { + private final Set added; + private final Set modified; + private final Set removed; + } + + @Data + public static class Diff { + private String filePath; + private DiffEntry.ChangeType changeType; + private String fileContentAtCommit1; + private String fileContentAtCommit2; + private String diffStringValue; + } + +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..d4f909df9e --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.sync.vc.BranchInfo; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.GitRepository.Diff; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +public interface GitRepositoryService { + + Set getActiveRepositoryTenants(); + + void prepareCommit(PendingCommit pendingCommit); + + PageData listVersions(TenantId tenantId, String branch, String path, PageLink pageLink) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String versionId, String path) throws Exception; + + void testRepository(TenantId tenantId, RepositorySettings settings) throws Exception; + + void initRepository(TenantId tenantId, RepositorySettings settings) throws Exception; + + RepositorySettings getRepositorySettings(TenantId tenantId) throws Exception; + + void clearRepository(TenantId tenantId) throws IOException; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void cleanUp(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; + + List getVersionsDiffList(TenantId tenantId, String path, String versionId1, String versionId2) throws IOException; + + String getContentsDiff(TenantId tenantId, String content1, String content2) throws IOException; + + void fetch(TenantId tenantId) throws GitAPIException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java new file mode 100644 index 0000000000..ccd5fc685e --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.UUID; + +@Data +public class PendingCommit { + + private final UUID txId; + private final String nodeId; + private final TenantId tenantId; + private final String workingBranch; + private String branch; + private String versionName; + + private String authorName; + + private String authorEmail; + + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { + this.tenantId = tenantId; + this.nodeId = nodeId; + this.txId = txId; + this.branch = branch; + this.versionName = versionName; + this.authorName = authorName; + this.authorEmail = authorEmail; + this.workingBranch = txId.toString(); + } +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java new file mode 100644 index 0000000000..26ab0d94c1 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlRequestCtx.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; + +import java.util.UUID; + +@RequiredArgsConstructor +@Data +public class VersionControlRequestCtx { + private final String nodeId; + private final UUID requestId; + private final TenantId tenantId; + private final RepositorySettings settings; + + public VersionControlRequestCtx(ToVersionControlServiceMsg msg, RepositorySettings settings) { + this.nodeId = msg.getNodeId(); + this.requestId = new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()); + this.tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); + this.settings = settings; + } + + @Override + public String toString() { + return "VersionControlRequestCtx{" + + "nodeId='" + nodeId + '\'' + + ", requestId=" + requestId + + ", tenantId=" + tenantId + + '}'; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 932a6e5ec1..663c19348d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao; import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import java.util.Collection; @@ -36,8 +37,12 @@ public interface Dao { T save(TenantId tenantId, T t); + T saveAndFlush(TenantId tenantId, T t); + boolean removeById(TenantId tenantId, UUID id); void removeAllByIds(Collection ids); + default EntityType getEntityType() { return null; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 4f250c47c8..4be3f8e6ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -31,6 +31,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; public abstract class DaoUtil { @@ -50,7 +52,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap()); } - public static Pageable toPageable(PageLink pageLink, Map columnMap) { + public static Pageable toPageable(PageLink pageLink, Map columnMap) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(pageLink.getSortOrder(), columnMap)); } @@ -58,7 +60,7 @@ public abstract class DaoUtil { return toPageable(pageLink, Collections.emptyMap(), sortOrders); } - public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { + public static Pageable toPageable(PageLink pageLink, Map columnMap, List sortOrders) { return PageRequest.of(pageLink.getPage(), pageLink.getPageSize(), pageLink.toSort(sortOrders, columnMap)); } @@ -107,4 +109,18 @@ public abstract class DaoUtil { return ids; } + public static void processInBatches(Function> finder, int batchSize, Consumer processor) { + PageLink pageLink = new PageLink(batchSize); + PageData batch; + + boolean hasNextBatch; + do { + batch = finder.apply(pageLink); + batch.getData().forEach(processor); + + hasNextBatch = batch.hasNext(); + pageLink = pageLink.nextPageLink(); + } while (hasNextBatch); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java new file mode 100644 index 0000000000..77800c31e8 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityDao.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; + +import java.util.UUID; + +public interface ExportableEntityDao> extends Dao { + + T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + + default T findByTenantIdAndName(UUID tenantId, String name) { throw new UnsupportedOperationException(); } + + PageData findByTenantId(UUID tenantId, PageLink pageLink); + + I getExternalIdByInternal(I internalId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java new file mode 100644 index 0000000000..6e2d2e115a --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ExportableEntityRepository.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao; + +import java.util.UUID; + +public interface ExportableEntityRepository { + + D findByTenantIdAndExternalId(UUID tenantId, UUID externalId); + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 0432e69bca..e3c51b48a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -19,10 +19,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -33,7 +35,7 @@ import java.util.UUID; * The Interface AssetDao. * */ -public interface AssetDao extends Dao, TenantEntityDao { +public interface AssetDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find asset info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 729c923e54..eb85f70687 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -23,8 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; @@ -53,7 +51,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; @@ -125,7 +122,7 @@ public class BaseAssetService extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(asset.getTenantId(), assetId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete asset that has entity views!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for assetId [{}]", assetId, e); - throw new RuntimeException("Exception while finding entity views for assetId [" + assetId + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(asset.getTenantId(), assetId); + if (entityViews != null && !entityViews.isEmpty()) { + throw new DataValidationException("Can't delete asset that has entity views!"); } publishEvictEvent(new AssetCacheEvictEvent(asset.getTenantId(), asset.getName(), null)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java index 3dc3f1e232..68a2a11c4a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeCacheKey.java @@ -34,6 +34,6 @@ public class AttributeCacheKey implements Serializable { @Override public String toString() { - return entityId + "_" + scope + "_" + key; + return "{" + entityId + "}" + scope + "_" + key; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java index d79537e564..0b2abcf517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerDao.java @@ -16,10 +16,12 @@ package org.thingsboard.server.dao.customer; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Optional; @@ -28,7 +30,7 @@ import java.util.UUID; /** * The Interface CustomerDao. */ -public interface CustomerDao extends Dao, TenantEntityDao { +public interface CustomerDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update customer object @@ -37,7 +39,7 @@ public interface CustomerDao extends Dao, TenantEntityDao { * @return saved customer object */ Customer save(TenantId tenantId, Customer customer); - + /** * Find customers by tenant id and page link. * diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index c79ec44880..547589169e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -31,7 +31,6 @@ import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -64,9 +63,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom @Autowired private DeviceService deviceService; - @Autowired - private EntityViewService entityViewService; - @Autowired private DashboardService dashboardService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java index dfe9152484..4246137d9a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardDao.java @@ -16,14 +16,19 @@ package org.thingsboard.server.dao.dashboard; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; +import java.util.List; +import java.util.UUID; + /** * The Interface DashboardDao. */ -public interface DashboardDao extends Dao, TenantEntityDao { +public interface DashboardDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Save or update dashboard object @@ -32,4 +37,7 @@ public interface DashboardDao extends Dao, TenantEntityDao { * @return saved dashboard object */ Dashboard save(TenantId tenantId, Dashboard dashboard); + + List findByTenantIdAndTitle(UUID tenantId, String title); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index ee5c44e20f..d76a3d675e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -40,6 +40,8 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import java.util.List; + import static org.thingsboard.server.dao.service.Validator.validateId; @Service @@ -56,7 +58,7 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb @Autowired private CustomerDao customerDao; - + @Autowired private EdgeDao edgeDao; @@ -279,19 +281,24 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb return dashboardInfoDao.findFirstByTenantIdAndName(tenantId.getId(), name); } + @Override + public List findTenantDashboardsByTitle(TenantId tenantId, String title) { + return dashboardDao.findByTenantIdAndTitle(tenantId.getId(), title); + } + private PaginatedRemover tenantDashboardsRemover = new PaginatedRemover() { - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink); - } + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return dashboardInfoDao.findDashboardsByTenantId(id.getId(), pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, DashboardInfo entity) { - deleteDashboard(tenantId, new DashboardId(entity.getUuidId())); - } - }; + @Override + protected void removeEntity(TenantId tenantId, DashboardInfo entity) { + deleteDashboard(tenantId, new DashboardId(entity.getUuidId())); + } + }; private class CustomerDashboardsUnassigner extends PaginatedRemover { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 85f66e4b67..c96f196d44 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -20,11 +20,13 @@ import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.List; @@ -35,7 +37,7 @@ import java.util.UUID; * The Interface DeviceDao. * */ -public interface DeviceDao extends Dao, TenantEntityDao { +public interface DeviceDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find device info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java index bf61cbdce5..4d4c73728f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java @@ -17,14 +17,16 @@ package org.thingsboard.server.dao.device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.UUID; -public interface DeviceProfileDao extends Dao { +public interface DeviceProfileDao extends Dao, ExportableEntityDao { DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, UUID deviceProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index df5a49c1e1..a6cfa33fb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,9 +18,8 @@ package org.thingsboard.server.dao.device; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -47,8 +46,6 @@ import org.thingsboard.server.dao.service.Validator; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -73,11 +70,10 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService deviceProfileValidator; + @Lazy @Autowired private QueueService queueService; - private final Lock findOrCreateLock = new ReentrantLock(); - @TransactionalEventListener(classes = DeviceProfileEvictEvent.class) @Override public void handleEvictEvent(DeviceProfileEvictEvent event) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index c486473e38..9f32939ad2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -78,7 +78,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; @@ -313,14 +312,9 @@ public class DeviceServiceImpl extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), deviceId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete device that has entity views!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for deviceId [{}]", deviceId, e); - throw new RuntimeException("Exception while finding entity views for deviceId [" + deviceId + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(device.getTenantId(), deviceId); + if (entityViews != null && !entityViews.isEmpty()) { + throw new DataValidationException("Can't delete device that has entity views!"); } DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, deviceId); @@ -523,14 +517,9 @@ public class DeviceServiceImpl extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); - if (!CollectionUtils.isEmpty(entityViews)) { - throw new DataValidationException("Can't assign device that has entity views to another tenant!"); - } - } catch (ExecutionException | InterruptedException e) { - log.error("Exception while finding entity views for deviceId [{}]", device.getId(), e); - throw new RuntimeException("Exception while finding entity views for deviceId [" + device.getId() + "]", e); + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(device.getTenantId(), device.getId()); + if (!CollectionUtils.isEmpty(entityViews)) { + throw new DataValidationException("Can't assign device that has entity views to another tenant!"); } eventService.removeEvents(device.getTenantId(), device.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 271c74a3e1..54bcace223 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.edge; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; @@ -26,9 +25,8 @@ import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; @@ -48,7 +46,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.rule.RuleChain; -import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.relation.RelationService; @@ -64,6 +62,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.toUUIDs; @@ -80,8 +79,6 @@ public class EdgeServiceImpl extends AbstractCachedEntityService findRelatedEdgeIdsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { log.trace("[{}] Executing findRelatedEdgeIdsByEntityId [{}] [{}]", tenantId, entityId, pageLink); - if (EntityType.TENANT.equals(entityId.getEntityType()) || - EntityType.CUSTOMER.equals(entityId.getEntityType()) || - EntityType.DEVICE_PROFILE.equals(entityId.getEntityType())) { - if (EntityType.TENANT.equals(entityId.getEntityType()) || - EntityType.DEVICE_PROFILE.equals(entityId.getEntityType())) { + switch (entityId.getEntityType()) { + case TENANT: + case DEVICE_PROFILE: + case OTA_PACKAGE: return convertToEdgeIds(findEdgesByTenantId(tenantId, pageLink)); - } else { + case CUSTOMER: return convertToEdgeIds(findEdgesByTenantIdAndCustomerId(tenantId, new CustomerId(entityId.getId()), pageLink)); - } - } else { - switch (entityId.getEntityType()) { - case EDGE: - List edgeIds = Collections.singletonList(new EdgeId(entityId.getId())); - return new PageData<>(edgeIds, 1, 1, false); - case DEVICE: - case ASSET: - case ENTITY_VIEW: - case DASHBOARD: - case RULE_CHAIN: - return convertToEdgeIds(findEdgesByTenantIdAndEntityId(tenantId, entityId, pageLink)); - case USER: - User userById = userService.findUserById(tenantId, new UserId(entityId.getId())); - if (userById == null) { - return createEmptyEdgeIdPageData(); - } - if (userById.getCustomerId() == null || userById.getCustomerId().isNullUid()) { - return convertToEdgeIds(findEdgesByTenantId(tenantId, pageLink)); - } else { - return convertToEdgeIds(findEdgesByTenantIdAndCustomerId(tenantId, userById.getCustomerId(), pageLink)); - } - default: - log.warn("[{}] Unsupported entity type {}", tenantId, entityId.getEntityType()); + case EDGE: + List edgeIds = Collections.singletonList(new EdgeId(entityId.getId())); + return new PageData<>(edgeIds, 1, 1, false); + case DEVICE: + case ASSET: + case ENTITY_VIEW: + case DASHBOARD: + case RULE_CHAIN: + return convertToEdgeIds(findEdgesByTenantIdAndEntityId(tenantId, entityId, pageLink)); + case USER: + User userById = userService.findUserById(tenantId, new UserId(entityId.getId())); + if (userById == null) { return createEmptyEdgeIdPageData(); - } + } + if (userById.getCustomerId() == null || userById.getCustomerId().isNullUid()) { + return convertToEdgeIds(findEdgesByTenantId(tenantId, pageLink)); + } else { + return convertToEdgeIds(findEdgesByTenantIdAndCustomerId(tenantId, userById.getCustomerId(), pageLink)); + } + default: + log.warn("[{}] Unsupported entity type {}", tenantId, entityId.getEntityType()); + return createEmptyEdgeIdPageData(); } } @@ -453,16 +445,19 @@ public class EdgeServiceImpl extends AbstractCachedEntityService edgeRuleChains = findEdgeRuleChains(tenantId, edgeId); List edgeRuleChainIds = edgeRuleChains.stream().map(IdBased::getId).collect(Collectors.toList()); - ObjectNode result = mapper.createObjectNode(); + ObjectNode result = JacksonUtil.OBJECT_MAPPER.createObjectNode(); for (RuleChain edgeRuleChain : edgeRuleChains) { - List connectionInfos = - ruleChainService.loadRuleChainMetaData(edgeRuleChain.getTenantId(), edgeRuleChain.getId()).getRuleChainConnections(); - if (connectionInfos != null && !connectionInfos.isEmpty()) { + List ruleNodes = + ruleChainService.loadRuleChainMetaData(edgeRuleChain.getTenantId(), edgeRuleChain.getId()).getNodes(); + if (ruleNodes != null && !ruleNodes.isEmpty()) { List connectedRuleChains = - connectionInfos.stream().map(RuleChainConnectionInfo::getTargetRuleChainId).collect(Collectors.toList()); + ruleNodes.stream() + .filter(rn -> rn.getType().equals(tbRuleChainInputNodeClassName)) + .map(rn -> new RuleChainId(UUID.fromString(rn.getConfiguration().get("ruleChainId").asText()))) + .collect(Collectors.toList()); List missingRuleChains = new ArrayList<>(); for (RuleChainId connectedRuleChain : connectedRuleChains) { if (!edgeRuleChainIds.contains(connectedRuleChain)) { @@ -471,7 +466,7 @@ public class EdgeServiceImpl extends AbstractCachedEntityService entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId).get(); - if (entityViews != null && !entityViews.isEmpty()) { - EntityView entityView = entityViews.get(0); - // TODO: @voba - refactor this blocking operation - Boolean relationExists = relationService.checkRelation(tenantId, edgeId, entityView.getId(), - EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE).get(); - if (relationExists) { - throw new DataValidationException("Can't unassign device/asset from edge that is related to entity view and entity view is assigned to edge!"); - } + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityId(tenantId, entityId); + if (entityViews != null && !entityViews.isEmpty()) { + EntityView entityView = entityViews.get(0); + Boolean relationExists = relationService.checkRelation( + tenantId, edgeId, entityView.getId(), + EntityRelation.CONTAINS_TYPE, RelationTypeGroup.EDGE + ); + if (relationExists) { + throw new DataValidationException("Can't unassign device/asset from edge that is related to entity view and entity view is assigned to edge!"); } - } catch (Exception e) { - log.error("[{}] Exception while finding entity views for entityId [{}]", tenantId, entityId, e); - throw new RuntimeException("Exception while finding entity views for entityId [" + entityId + "]", e); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java index f44bafe937..75a9c122e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewDao.java @@ -19,10 +19,12 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.id.EntityViewId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import java.util.List; import java.util.Optional; @@ -31,7 +33,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/28/2017. */ -public interface EntityViewDao extends Dao { +public interface EntityViewDao extends Dao, ExportableEntityDao { /** * Find entity view info by id. diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 04bdb6697a..6b1d6775d6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -22,8 +22,6 @@ import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; @@ -53,7 +51,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; -import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -68,7 +65,6 @@ import static org.thingsboard.server.dao.service.Validator.validateString; public class EntityViewServiceImpl extends AbstractCachedEntityService implements EntityViewService { public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - public static final String INCORRECT_PAGE_LINK = "Incorrect page link "; public static final String INCORRECT_CUSTOMER_ID = "Incorrect customerId "; public static final String INCORRECT_ENTITY_VIEW_ID = "Incorrect entityViewId "; public static final String INCORRECT_EDGE_ID = "Incorrect edgeId "; @@ -280,6 +276,17 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService new EntityViewCacheValue(null, v), true)); } + @Override + public List findEntityViewsByTenantIdAndEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findEntityViewsByTenantIdAndEntityId, tenantId [{}], entityId [{}]", tenantId, entityId); + validateId(tenantId, INCORRECT_TENANT_ID + tenantId); + validateId(entityId.getId(), "Incorrect entityId" + entityId); + + return cache.getAndPutInTransaction(EntityViewCacheKey.byEntityId(tenantId, entityId), + () -> entityViewDao.findEntityViewsByTenantIdAndEntityId(tenantId.getId(), entityId.getId()), + EntityViewCacheValue::getEntityViews, v -> new EntityViewCacheValue(null, v), true); + } + @Override public void deleteEntityView(TenantId tenantId, EntityViewId entityViewId) { log.trace("Executing deleteEntityView [{}]", entityViewId); @@ -320,15 +327,10 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService implements BaseEntity { @Column(name = ModelConstants.ID_PROPERTY, columnDefinition = "uuid") protected UUID id; - @Column(name = ModelConstants.CREATED_TIME_PROPERTY) + @Column(name = ModelConstants.CREATED_TIME_PROPERTY, updatable = false) protected long createdTime; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index f6f838eb62..80e21a3fae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -90,6 +90,8 @@ public class ModelConstants { * Cassandra admin_settings constants. */ public static final String ADMIN_SETTINGS_COLUMN_FAMILY_NAME = "admin_settings"; + + public static final String ADMIN_SETTINGS_TENANT_ID_PROPERTY = TENANT_ID_PROPERTY; public static final String ADMIN_SETTINGS_KEY_PROPERTY = "key"; public static final String ADMIN_SETTINGS_JSON_VALUE_PROPERTY = "json_value"; @@ -559,6 +561,8 @@ public class ModelConstants { public static final String EDGE_EVENT_BY_ID_VIEW_NAME = "edge_event_by_id"; + public static final String EXTERNAL_ID_PROPERTY = "external_id"; + /** * User auth settings constants. * */ diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java index 4405768f7c..0936ca2855 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractAssetEntity.java @@ -38,6 +38,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.ASSET_LABEL_PROPER import static org.thingsboard.server.dao.model.ModelConstants.ASSET_NAME_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TENANT_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ASSET_TYPE_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.EXTERNAL_ID_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.SEARCH_TEXT_PROPERTY; @Data @@ -68,6 +69,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity @Column(name = ModelConstants.ASSET_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = EXTERNAL_ID_PROPERTY) + private UUID externalId; + public AbstractAssetEntity() { super(); } @@ -87,6 +91,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.type = asset.getType(); this.label = asset.getLabel(); this.additionalInfo = asset.getAdditionalInfo(); + if (asset.getExternalId() != null) { + this.externalId = asset.getExternalId().getId(); + } } public AbstractAssetEntity(AssetEntity assetEntity) { @@ -99,6 +106,7 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity this.label = assetEntity.getLabel(); this.searchText = assetEntity.getSearchText(); this.additionalInfo = assetEntity.getAdditionalInfo(); + this.externalId = assetEntity.getExternalId(); } @Override @@ -128,6 +136,9 @@ public abstract class AbstractAssetEntity extends BaseSqlEntity asset.setType(type); asset.setLabel(label); asset.setAdditionalInfo(additionalInfo); + if (externalId != null) { + asset.setExternalId(new AssetId(externalId)); + } return asset; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java index 717ba1f9e3..dbc0a057ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractDeviceEntity.java @@ -84,6 +84,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti @Column(name = ModelConstants.DEVICE_DEVICE_DATA_PROPERTY, columnDefinition = "jsonb") private JsonNode deviceData; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY, columnDefinition = "uuid") + private UUID externalId; + public AbstractDeviceEntity() { super(); } @@ -113,6 +116,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.type = device.getType(); this.label = device.getLabel(); this.additionalInfo = device.getAdditionalInfo(); + if (device.getExternalId() != null) { + this.externalId = device.getExternalId().getId(); + } } public AbstractDeviceEntity(DeviceEntity deviceEntity) { @@ -129,6 +135,7 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti this.additionalInfo = deviceEntity.getAdditionalInfo(); this.firmwareId = deviceEntity.getFirmwareId(); this.softwareId = deviceEntity.getSoftwareId(); + this.externalId = deviceEntity.getExternalId(); } @Override @@ -164,6 +171,9 @@ public abstract class AbstractDeviceEntity extends BaseSqlEnti device.setType(type); device.setLabel(label); device.setAdditionalInfo(additionalInfo); + if (externalId != null) { + device.setExternalId(new DeviceId(externalId)); + } return device; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java index a8a31cd892..cc68adb2a1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AbstractEntityViewEntity.java @@ -89,6 +89,9 @@ public abstract class AbstractEntityViewEntity extends Bas @Column(name = ModelConstants.ENTITY_VIEW_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + private static final ObjectMapper mapper = new ObjectMapper(); public AbstractEntityViewEntity() { @@ -121,6 +124,9 @@ public abstract class AbstractEntityViewEntity extends Bas this.endTs = entityView.getEndTimeMs(); this.searchText = entityView.getSearchText(); this.additionalInfo = entityView.getAdditionalInfo(); + if (entityView.getExternalId() != null) { + this.externalId = entityView.getExternalId().getId(); + } } public AbstractEntityViewEntity(EntityViewEntity entityViewEntity) { @@ -137,6 +143,7 @@ public abstract class AbstractEntityViewEntity extends Bas this.endTs = entityViewEntity.getEndTs(); this.searchText = entityViewEntity.getSearchText(); this.additionalInfo = entityViewEntity.getAdditionalInfo(); + this.externalId = entityViewEntity.getExternalId(); } @Override @@ -172,6 +179,9 @@ public abstract class AbstractEntityViewEntity extends Bas entityView.setStartTimeMs(startTs); entityView.setEndTimeMs(endTs); entityView.setAdditionalInfo(additionalInfo); + if (externalId != null) { + entityView.setExternalId(new EntityViewId(externalId)); + } return entityView; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java index 49d5002af2..4da17d2c34 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/AdminSettingsEntity.java @@ -22,14 +22,18 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseEntity; import org.thingsboard.server.dao.model.BaseSqlEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; import javax.persistence.Table; +import java.util.UUID; + import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_COLUMN_FAMILY_NAME; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_JSON_VALUE_PROPERTY; import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY_PROPERTY; @@ -41,6 +45,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.ADMIN_SETTINGS_KEY @Table(name = ADMIN_SETTINGS_COLUMN_FAMILY_NAME) public final class AdminSettingsEntity extends BaseSqlEntity implements BaseEntity { + @Column(name = ModelConstants.ADMIN_SETTINGS_TENANT_ID_PROPERTY) + private UUID tenantId; + @Column(name = ADMIN_SETTINGS_KEY_PROPERTY) private String key; @@ -57,6 +64,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl this.setUuid(adminSettings.getId().getId()); } this.setCreatedTime(adminSettings.getCreatedTime()); + this.tenantId = adminSettings.getTenantId().getId(); this.key = adminSettings.getKey(); this.jsonValue = adminSettings.getJsonValue(); } @@ -65,6 +73,7 @@ public final class AdminSettingsEntity extends BaseSqlEntity impl public AdminSettings toData() { AdminSettings adminSettings = new AdminSettings(new AdminSettingsId(id)); adminSettings.setCreatedTime(createdTime); + adminSettings.setTenantId(TenantId.fromUUID(tenantId)); adminSettings.setKey(key); adminSettings.setJsonValue(jsonValue); return adminSettings; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java index 61b21ecc4f..3dec26afd9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CustomerEntity.java @@ -77,6 +77,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea @Column(name = ModelConstants.CUSTOMER_ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public CustomerEntity() { super(); } @@ -97,6 +100,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea this.phone = customer.getPhone(); this.email = customer.getEmail(); this.additionalInfo = customer.getAdditionalInfo(); + if (customer.getExternalId() != null) { + this.externalId = customer.getExternalId().getId(); + } } @Override @@ -124,6 +130,9 @@ public final class CustomerEntity extends BaseSqlEntity implements Sea customer.setPhone(phone); customer.setEmail(email); customer.setAdditionalInfo(additionalInfo); + if (externalId != null) { + customer.setExternalId(new CustomerId(externalId)); + } return customer; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java index 45b4bf9210..ccaea5524e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DashboardEntity.java @@ -78,6 +78,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S @Column(name = ModelConstants.DASHBOARD_CONFIGURATION_PROPERTY) private JsonNode configuration; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DashboardEntity() { super(); } @@ -102,6 +105,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S this.mobileHide = dashboard.isMobileHide(); this.mobileOrder = dashboard.getMobileOrder(); this.configuration = dashboard.getConfiguration(); + if (dashboard.getExternalId() != null) { + this.externalId = dashboard.getExternalId().getId(); + } } @Override @@ -133,6 +139,9 @@ public final class DashboardEntity extends BaseSqlEntity implements S dashboard.setMobileHide(mobileHide); dashboard.setMobileOrder(mobileOrder); dashboard.setConfiguration(configuration); + if (externalId != null) { + dashboard.setExternalId(new DashboardId(externalId)); + } return dashboard; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java index 9dc5c9780e..db1a04f746 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/DeviceProfileEntity.java @@ -104,6 +104,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl @Column(name = ModelConstants.DEVICE_PROFILE_SOFTWARE_ID_PROPERTY) private UUID softwareId; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public DeviceProfileEntity() { super(); } @@ -140,6 +143,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (deviceProfile.getSoftwareId() != null) { this.softwareId = deviceProfile.getSoftwareId().getId(); } + if (deviceProfile.getExternalId() != null) { + this.externalId = deviceProfile.getExternalId().getId(); + } } @Override @@ -189,6 +195,9 @@ public final class DeviceProfileEntity extends BaseSqlEntity impl if (softwareId != null) { deviceProfile.setSoftwareId(new OtaPackageId(softwareId)); } + if (externalId != null) { + deviceProfile.setExternalId(new DeviceProfileId(externalId)); + } return deviceProfile; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java index 09ed75b5e5..88c8c79fdc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleChainEntity.java @@ -75,6 +75,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT @Column(name = ModelConstants.ADDITIONAL_INFO_PROPERTY) private JsonNode additionalInfo; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleChainEntity() { } @@ -94,6 +97,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT this.debugMode = ruleChain.isDebugMode(); this.configuration = ruleChain.getConfiguration(); this.additionalInfo = ruleChain.getAdditionalInfo(); + if (ruleChain.getExternalId() != null) { + this.externalId = ruleChain.getExternalId().getId(); + } } @Override @@ -120,6 +126,9 @@ public class RuleChainEntity extends BaseSqlEntity implements SearchT ruleChain.setDebugMode(debugMode); ruleChain.setConfiguration(configuration); ruleChain.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleChain.setExternalId(new RuleChainId(externalId)); + } return ruleChain; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java index 04b7fd4ecb..70e8497c31 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/RuleNodeEntity.java @@ -64,6 +64,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex @Column(name = ModelConstants.DEBUG_MODE) private boolean debugMode; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public RuleNodeEntity() { } @@ -81,6 +84,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex this.searchText = ruleNode.getName(); this.configuration = ruleNode.getConfiguration(); this.additionalInfo = ruleNode.getAdditionalInfo(); + if (ruleNode.getExternalId() != null) { + this.externalId = ruleNode.getExternalId().getId(); + } } @Override @@ -105,6 +111,9 @@ public class RuleNodeEntity extends BaseSqlEntity implements SearchTex ruleNode.setDebugMode(debugMode); ruleNode.setConfiguration(configuration); ruleNode.setAdditionalInfo(additionalInfo); + if (externalId != null) { + ruleNode.setExternalId(new RuleNodeId(externalId)); + } return ruleNode; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java index 5887c6f61e..cbd6068d7f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/WidgetsBundleEntity.java @@ -54,6 +54,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl @Column(name = ModelConstants.WIDGETS_BUNDLE_DESCRIPTION) private String description; + @Column(name = ModelConstants.EXTERNAL_ID_PROPERTY) + private UUID externalId; + public WidgetsBundleEntity() { super(); } @@ -70,6 +73,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl this.title = widgetsBundle.getTitle(); this.image = widgetsBundle.getImage(); this.description = widgetsBundle.getDescription(); + if (widgetsBundle.getExternalId() != null) { + this.externalId = widgetsBundle.getExternalId().getId(); + } } @Override @@ -93,6 +99,9 @@ public final class WidgetsBundleEntity extends BaseSqlEntity impl widgetsBundle.setTitle(title); widgetsBundle.setImage(image); widgetsBundle.setDescription(description); + if (externalId != null) { + widgetsBundle.setExternalId(new WidgetsBundleId(externalId)); + } return widgetsBundle; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 4d0bb18a43..1587aa9f52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -22,8 +22,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; -import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.common.data.OtaPackage; @@ -42,8 +40,6 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.List; import java.util.Optional; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -248,8 +244,4 @@ public class BaseOtaPackageService extends AbstractCachedEntityService toOtaPackageInfoKey(OtaPackageId otaPackageId) { - return Collections.singletonList(otaPackageId); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java index a0254a4923..45b5b2cc7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.QueueId; @@ -43,6 +44,7 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ @Autowired private QueueDao queueDao; + @Lazy @Autowired private TbTenantProfileCache tenantProfileCache; diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 912a3393c4..f60920a31d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.relation; import com.google.common.base.Function; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -24,12 +25,11 @@ import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.dao.ConcurrencyFailureException; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.util.StringUtils; import org.thingsboard.server.cache.TbTransactionalCache; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; @@ -90,7 +90,14 @@ public class BaseRelationService implements RelationService { } @Override - public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + log.trace("Executing checkRelationAsync [{}][{}][{}][{}]", from, to, relationType, typeGroup); + validate(from, to, relationType, typeGroup); + return relationDao.checkRelationAsync(tenantId, from, to, relationType, typeGroup); + } + + @Override + public boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { log.trace("Executing checkRelation [{}][{}][{}][{}]", from, to, relationType, typeGroup); validate(from, to, relationType, typeGroup); return relationDao.checkRelation(tenantId, from, to, relationType, typeGroup); @@ -119,6 +126,20 @@ public class BaseRelationService implements RelationService { return result; } + @Override + public void saveRelations(TenantId tenantId, List relations) { + log.trace("Executing saveRelations [{}]", relations); + for (EntityRelation relation : relations) { + validate(relation); + } + for (List partition : Lists.partition(relations, 1024)) { + relationDao.saveRelations(tenantId, partition); + } + for (EntityRelation relation : relations) { + publishEvictEvent(EntityRelationEvent.from(relation)); + } + } + @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { log.trace("Executing saveRelationAsync [{}]", relation); @@ -167,68 +188,33 @@ public class BaseRelationService implements RelationService { return future; } + @Transactional @Override public void deleteEntityRelations(TenantId tenantId, EntityId entityId) { log.trace("Executing deleteEntityRelations [{}]", entityId); validate(entityId); - List inboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelations.addAll(relationDao.findAllByTo(tenantId, entityId, typeGroup)); - } - - List outboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelations.addAll(relationDao.findAllByFrom(tenantId, entityId, typeGroup)); - } - - for (EntityRelation relation : inboundRelations) { - delete(tenantId, relation, true); - } + List inboundRelations = new ArrayList<>(relationDao.findAllByTo(tenantId, entityId)); + List outboundRelations = new ArrayList<>(relationDao.findAllByFrom(tenantId, entityId)); - for (EntityRelation relation : outboundRelations) { - delete(tenantId, relation, false); - } - - relationDao.deleteOutboundRelations(tenantId, entityId); - - } + if (!inboundRelations.isEmpty()) { + try { + relationDao.deleteInboundRelations(tenantId, entityId); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", inboundRelations, e); + } - @Override - public ListenableFuture deleteEntityRelationsAsync(TenantId tenantId, EntityId entityId) { - log.trace("Executing deleteEntityRelationsAsync [{}]", entityId); - validate(entityId); - List>> inboundRelationsList = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelationsList.add(executor.submit(() -> relationDao.findAllByTo(tenantId, entityId, typeGroup))); + for (EntityRelation relation : inboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } } - ListenableFuture>> inboundRelations = Futures.allAsList(inboundRelationsList); + if (!outboundRelations.isEmpty()) { + relationDao.deleteOutboundRelations(tenantId, entityId); - List>> outboundRelationsList = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelationsList.add(executor.submit(() -> relationDao.findAllByFrom(tenantId, entityId, typeGroup))); + for (EntityRelation relation : outboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } } - - ListenableFuture>> outboundRelations = Futures.allAsList(outboundRelationsList); - - ListenableFuture> inboundDeletions = Futures.transformAsync(inboundRelations, - relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, true); - return Futures.allAsList(results); - }, MoreExecutors.directExecutor()); - - ListenableFuture> outboundDeletions = Futures.transformAsync(outboundRelations, - relations -> { - List> results = deleteRelationGroupsAsync(tenantId, relations, false); - return Futures.allAsList(results); - }, MoreExecutors.directExecutor()); - - ListenableFuture>> deletionsFuture = Futures.allAsList(inboundDeletions, outboundDeletions); - - return Futures.transform(Futures.transformAsync(deletionsFuture, - (deletions) -> relationDao.deleteOutboundRelationsAsync(tenantId, entityId), - MoreExecutors.directExecutor()), - result -> null, MoreExecutors.directExecutor()); } private List> deleteRelationGroupsAsync(TenantId tenantId, List> relations, boolean deleteFromDb) { @@ -252,18 +238,6 @@ public class BaseRelationService implements RelationService { } } - boolean delete(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { - eventPublisher.publishEvent(EntityRelationEvent.from(relation)); - if (deleteFromDb) { - try { - return relationDao.deleteRelation(tenantId, relation); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", relation, e); - } - } - return false; - } - @Override public List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { validate(from); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index e15f4f3d69..a3e81c2652 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -22,6 +22,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChainType; +import java.util.Collection; import java.util.List; /** @@ -31,18 +32,26 @@ public interface RelationDao { List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); + List findAllByFrom(TenantId tenantId, EntityId from); + List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + List findAllByTo(TenantId tenantId, EntityId to); + List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); - ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); + + boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); EntityRelation getRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); boolean saveRelation(TenantId tenantId, EntityRelation relation); + void saveRelations(TenantId tenantId, Collection relations); + ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation); boolean deleteRelation(TenantId tenantId, EntityRelation relation); @@ -53,7 +62,9 @@ public interface RelationDao { ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean deleteOutboundRelations(TenantId tenantId, EntityId entity); + void deleteOutboundRelations(TenantId tenantId, EntityId entity); + + void deleteInboundRelations(TenantId tenantId, EntityId entity); ListenableFuture deleteOutboundRelationsAsync(TenantId tenantId, EntityId entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 4be7227214..28aa798c53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -58,6 +58,8 @@ import org.thingsboard.server.dao.service.Validator; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -68,7 +70,9 @@ import java.util.Set; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.TENANT; -import static org.thingsboard.server.dao.service.Validator.*; +import static org.thingsboard.server.dao.service.Validator.validateId; +import static org.thingsboard.server.dao.service.Validator.validatePageLink; +import static org.thingsboard.server.dao.service.Validator.validateString; /** * Created by igor on 3/12/18. @@ -139,6 +143,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List nodes = ruleChainMetaData.getNodes(); List toAddOrUpdate = new ArrayList<>(); List toDelete = new ArrayList<>(); + List relations = new ArrayList<>(); Map ruleNodeIndexMap = new HashMap<>(); if (nodes != null) { @@ -169,15 +174,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (RuleNode node : toAddOrUpdate) { node.setRuleChainId(ruleChain.getId()); RuleNode savedNode = ruleNodeDao.save(tenantId, node); - createRelation(tenantId, new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), + relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); int index = nodes.indexOf(node); nodes.set(index, savedNode); ruleNodeIndexMap.put(savedNode.getId(), index); } } - for (RuleNode node : toDelete) { - deleteRuleNode(tenantId, node.getId()); + if (!toDelete.isEmpty()) { + deleteRuleNodes(tenantId, toDelete); } RuleNodeId firstRuleNodeId = null; if (nodes != null) { @@ -194,7 +199,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC EntityId from = nodes.get(nodeConnection.getFromIndex()).getId(); EntityId to = nodes.get(nodeConnection.getToIndex()).getId(); String type = nodeConnection.getType(); - createRelation(tenantId, new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); + relations.add(new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); } } if (ruleChainMetaData.getRuleChainConnections() != null) { @@ -220,7 +225,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleChainToRuleNode.setTo(targetNode.getId()); sourceRuleChainToRuleNode.setType(EntityRelation.CONTAINS_TYPE); sourceRuleChainToRuleNode.setTypeGroup(RelationTypeGroup.RULE_CHAIN); - relationService.saveRelation(tenantId, sourceRuleChainToRuleNode); + relations.add(sourceRuleChainToRuleNode); EntityRelation sourceRuleNodeToTargetRuleNode = new EntityRelation(); EntityId from = nodes.get(nodeToRuleChainConnection.getFromIndex()).getId(); @@ -228,11 +233,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleNodeToTargetRuleNode.setTo(targetNode.getId()); sourceRuleNodeToTargetRuleNode.setType(nodeToRuleChainConnection.getType()); sourceRuleNodeToTargetRuleNode.setTypeGroup(RelationTypeGroup.RULE_NODE); - relationService.saveRelation(tenantId, sourceRuleNodeToTargetRuleNode); - } + relations.add(sourceRuleNodeToTargetRuleNode); + } } } + if (!relations.isEmpty()) { + relationService.saveRelations(tenantId, relations); + } + return RuleChainUpdateResult.successful(updatedRuleNodes); } @@ -271,6 +280,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC RuleChainMetaData ruleChainMetaData = new RuleChainMetaData(); ruleChainMetaData.setRuleChainId(ruleChainId); List ruleNodes = getRuleChainNodes(tenantId, ruleChainId); + Collections.sort(ruleNodes, Comparator.comparingLong(RuleNode::getCreatedTime).thenComparing(RuleNode::getId, Comparator.comparing(RuleNodeId::getId))); Map ruleNodeIndexMap = new HashMap<>(); for (RuleNode node : ruleNodes) { ruleNodeIndexMap.put(node.getId(), ruleNodes.indexOf(node)); @@ -293,6 +303,10 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } } } + if (ruleChainMetaData.getConnections() != null) { + Collections.sort(ruleChainMetaData.getConnections(), Comparator.comparingInt(NodeConnectionInfo::getFromIndex) + .thenComparing(NodeConnectionInfo::getToIndex).thenComparing(NodeConnectionInfo::getType)); + } return ruleChainMetaData; } @@ -390,6 +404,11 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return ruleChainDao.findRuleChainsByTenantIdAndType(tenantId.getId(), type, pageLink); } + @Override + public Collection findTenantRuleChainsByTypeAndName(TenantId tenantId, RuleChainType type, String name) { + return ruleChainDao.findByTenantIdAndTypeAndName(tenantId, type, name); + } + @Override @Transactional public void deleteRuleChainById(TenantId tenantId, RuleChainId ruleChainId) { @@ -456,7 +475,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC ruleChain.setRoot(false); if (overwrite) { - Collection existingRuleChains = ruleChainDao.findByTenantIdAndTypeAndName(tenantId, + Collection existingRuleChains = findTenantRuleChainsByTypeAndName(tenantId, Optional.ofNullable(ruleChain.getType()).orElse(RuleChainType.CORE), ruleChain.getName()); Optional existingRuleChain = existingRuleChains.stream().findFirst(); if (existingRuleChain.isPresent()) { @@ -699,6 +718,19 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC throw t; } } + deleteRuleNodes(tenantId, ruleChainId); + } + + private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { + List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + for (var node : ruleNodes) { + deleteEntityRelations(tenantId, node.getId()); + } + ruleNodeDao.deleteByIdIn(ruleNodeIds); + } + + @Override + public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { List nodeRelations = getRuleChainToNodeRelations(tenantId, ruleChainId); for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java index f53f8a9e9a..d19074aa53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleChainDao.java @@ -15,12 +15,14 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityDao; import java.util.Collection; @@ -29,7 +31,7 @@ import java.util.UUID; /** * Created by igor on 3/12/18. */ -public interface RuleChainDao extends Dao, TenantEntityDao { +public interface RuleChainDao extends Dao, TenantEntityDao, ExportableEntityDao { /** * Find rule chains by tenantId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java index b4f087b98a..4fb15e5721 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,4 +33,8 @@ public interface RuleNodeDao extends Dao { List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search); PageData findAllRuleNodesByType(String type, PageLink pageLink); + + List findByExternalIds(RuleChainId ruleChainId, List externalIds); + + void deleteByIdIn(List ruleNodeIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java index 5bbe636dc7..a95f7a43aa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AlarmDataValidator.java @@ -18,18 +18,17 @@ package org.thingsboard.server.dao.service.validator; import lombok.AllArgsConstructor; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component @AllArgsConstructor public class AlarmDataValidator extends DataValidator { - private final TenantDao tenantDao; + private final TenantService tenantService; @Override protected void validateDataImpl(TenantId tenantId, Alarm alarm) { @@ -48,8 +47,7 @@ public class AlarmDataValidator extends DataValidator { if (alarm.getTenantId() == null) { throw new DataValidationException("Alarm should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(alarm.getTenantId(), alarm.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(alarm.getTenantId())) { throw new DataValidationException("Alarm is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java index a89acb9631..28e4dc0582 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ApiUsageDataValidator.java @@ -15,29 +15,29 @@ */ package org.thingsboard.server.dao.service.validator; -import lombok.AllArgsConstructor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component -@AllArgsConstructor public class ApiUsageDataValidator extends DataValidator { - private final TenantDao tenantDao; + @Lazy + @Autowired + private TenantService tenantService; @Override protected void validateDataImpl(TenantId requestTenantId, ApiUsageState apiUsageState) { if (apiUsageState.getTenantId() == null) { throw new DataValidationException("ApiUsageState should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(requestTenantId, apiUsageState.getTenantId().getId()); - if (tenant == null && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { + if (!tenantService.tenantExists(apiUsageState.getTenantId()) && !requestTenantId.equals(TenantId.SYS_TENANT_ID)) { throw new DataValidationException("ApiUsageState is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java index a2204491f9..1f303cc79d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AssetDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,7 +31,7 @@ import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -43,7 +42,7 @@ public class AssetDataValidator extends DataValidator { private AssetDao assetDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -82,8 +81,7 @@ public class AssetDataValidator extends DataValidator { if (asset.getTenantId() == null) { throw new DataValidationException("Asset should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, asset.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(asset.getTenantId())) { throw new DataValidationException("Asset is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java index 8184ff8e8e..c7ffe8c61e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/BaseOtaPackageDataValidator.java @@ -20,18 +20,17 @@ import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.Objects; public abstract class BaseOtaPackageDataValidator> extends DataValidator { @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private DeviceProfileDao deviceProfileDao; @@ -40,8 +39,7 @@ public abstract class BaseOtaPackageDataValidator> extends if (otaPackageInfo.getTenantId() == null) { throw new DataValidationException("OtaPackage should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(otaPackageInfo.getTenantId(), otaPackageInfo.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(otaPackageInfo.getTenantId())) { throw new DataValidationException("OtaPackage is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java index b8e1067887..9cc8d6a0de 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/CustomerDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.customer.CustomerDao; @@ -29,7 +28,7 @@ import org.thingsboard.server.dao.customer.CustomerServiceImpl; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.Optional; @@ -40,7 +39,7 @@ public class CustomerDataValidator extends DataValidator { private CustomerDao customerDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -87,8 +86,7 @@ public class CustomerDataValidator extends DataValidator { if (customer.getTenantId() == null) { throw new DataValidationException("Customer should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, customer.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(customer.getTenantId())) { throw new DataValidationException("Customer is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java index ee226af853..23b960253c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DashboardDataValidator.java @@ -21,14 +21,13 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component public class DashboardDataValidator extends DataValidator { @@ -37,7 +36,7 @@ public class DashboardDataValidator extends DataValidator { private DashboardDao dashboardDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -59,8 +58,7 @@ public class DashboardDataValidator extends DataValidator { if (dashboard.getTenantId() == null) { throw new DataValidationException("Dashboard should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, dashboard.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(dashboard.getTenantId())) { throw new DataValidationException("Dashboard is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java index e9d8b82e1c..23e520f070 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,7 +34,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.Optional; @@ -48,7 +47,7 @@ public class DeviceDataValidator extends DataValidator { private DeviceDao deviceDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired private CustomerDao customerDao; @@ -85,8 +84,7 @@ public class DeviceDataValidator extends DataValidator { if (device.getTenantId() == null) { throw new DataValidationException("Device should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(device.getTenantId(), device.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(device.getTenantId())) { throw new DataValidationException("Device is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index f04cbd0226..b1fb9b150b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -37,7 +37,6 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -67,7 +66,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import java.util.HashSet; import java.util.List; @@ -90,9 +89,9 @@ public class DeviceProfileDataValidator extends DataValidator { @Autowired private DeviceDao deviceDao; @Autowired - private TenantDao tenantDao; - @Autowired + private TenantService tenantService; @Lazy + @Autowired private QueueService queueService; @Autowired private OtaPackageService otaPackageService; @@ -119,8 +118,7 @@ public class DeviceProfileDataValidator extends DataValidator { if (deviceProfile.getTenantId() == null) { throw new DataValidationException("Device profile should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(deviceProfile.getTenantId(), deviceProfile.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(deviceProfile.getTenantId())) { throw new DataValidationException("Device profile is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java index 8289d21b23..b88724f7af 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EdgeDataValidator.java @@ -19,7 +19,6 @@ import lombok.AllArgsConstructor; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +26,7 @@ import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.edge.EdgeDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -36,7 +35,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public class EdgeDataValidator extends DataValidator { private final EdgeDao edgeDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final CustomerDao customerDao; @Override @@ -65,8 +64,7 @@ public class EdgeDataValidator extends DataValidator { if (edge.getTenantId() == null) { throw new DataValidationException("Edge should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(edge.getTenantId(), edge.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(edge.getTenantId())) { throw new DataValidationException("Edge is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java index a38850c63c..7532dda195 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/EntityViewDataValidator.java @@ -20,14 +20,13 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.entityview.EntityViewDao; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -36,7 +35,7 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public class EntityViewDataValidator extends DataValidator { private final EntityViewDao entityViewDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final CustomerDao customerDao; @Override @@ -69,8 +68,7 @@ public class EntityViewDataValidator extends DataValidator { if (entityView.getTenantId() == null) { throw new DataValidationException("Entity view should be assigned to tenant!"); } else { - Tenant tenant = tenantDao.findById(tenantId, entityView.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(entityView.getTenantId())) { throw new DataValidationException("Entity view is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java index 90ac54577a..49de43dba3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TbResource; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; @@ -28,7 +27,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.resource.TbResourceDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import static org.thingsboard.server.common.data.EntityType.TB_RESOURCE; @@ -39,7 +38,7 @@ public class ResourceDataValidator extends DataValidator { private TbResourceDao resourceDao; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -73,8 +72,7 @@ public class ResourceDataValidator extends DataValidator { resource.setTenantId(TenantId.fromUUID(ModelConstants.NULL_UUID)); } if (!resource.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, resource.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(resource.getTenantId())) { throw new DataValidationException("Resource is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java index f36b4d23fa..1581a61b80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/RuleChainDataValidator.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; @@ -30,7 +29,7 @@ import org.thingsboard.server.dao.rule.RuleChainDao; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; @Component public class RuleChainDataValidator extends DataValidator { @@ -43,7 +42,7 @@ public class RuleChainDataValidator extends DataValidator { private RuleChainService ruleChainService; @Autowired - private TenantDao tenantDao; + private TenantService tenantService; @Autowired @Lazy @@ -68,8 +67,7 @@ public class RuleChainDataValidator extends DataValidator { if (ruleChain.getTenantId() == null || ruleChain.getTenantId().isNullUid()) { throw new DataValidationException("Rule chain should be assigned to tenant!"); } - Tenant tenant = tenantDao.findById(tenantId, ruleChain.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(ruleChain.getTenantId())) { throw new DataValidationException("Rule chain is referencing to non-existent tenant!"); } if (ruleChain.isRoot() && RuleChainType.CORE.equals(ruleChain.getType())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java index 94b894acce..21e2143f1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java @@ -21,7 +21,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -32,7 +31,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserDao; import org.thingsboard.server.dao.user.UserService; @@ -46,9 +45,6 @@ public class UserDataValidator extends DataValidator { @Lazy private UserService userService; - @Autowired - private TenantDao tenantDao; - @Autowired private CustomerDao customerDao; @@ -56,6 +52,10 @@ public class UserDataValidator extends DataValidator { @Lazy private TbTenantProfileCache tenantProfileCache; + @Autowired + @Lazy + private TenantService tenantService; + @Override protected void validateCreate(TenantId tenantId, User user) { if (!user.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { @@ -119,8 +119,7 @@ public class UserDataValidator extends DataValidator { + " already present in database!"); } if (!tenantId.getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, user.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(user.getTenantId())) { throw new DataValidationException("User is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java index 93a1766546..25c6b5db86 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetTypeDataValidator.java @@ -27,6 +27,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.widget.WidgetTypeDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; @@ -35,8 +36,8 @@ import org.thingsboard.server.dao.widget.WidgetsBundleDao; public class WidgetTypeDataValidator extends DataValidator { private final WidgetTypeDao widgetTypeDao; - private final TenantDao tenantDao; private final WidgetsBundleDao widgetsBundleDao; + private final TenantService tenantService; @Override protected void validateDataImpl(TenantId tenantId, WidgetTypeDetails widgetTypeDetails) { @@ -53,8 +54,7 @@ public class WidgetTypeDataValidator extends DataValidator { widgetTypeDetails.setTenantId(TenantId.fromUUID(ModelConstants.NULL_UUID)); } if (!widgetTypeDetails.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, widgetTypeDetails.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(widgetTypeDetails.getTenantId())) { throw new DataValidationException("Widget type is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java index b140dfc79d..b950a12826 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/WidgetsBundleDataValidator.java @@ -18,13 +18,12 @@ package org.thingsboard.server.dao.service.validator; import lombok.AllArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.widget.WidgetsBundleDao; @Component @@ -32,7 +31,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleDao; public class WidgetsBundleDataValidator extends DataValidator { private final WidgetsBundleDao widgetsBundleDao; - private final TenantDao tenantDao; + private final TenantService tenantService; @Override protected void validateDataImpl(TenantId tenantId, WidgetsBundle widgetsBundle) { @@ -43,8 +42,7 @@ public class WidgetsBundleDataValidator extends DataValidator { widgetsBundle.setTenantId(TenantId.fromUUID(ModelConstants.NULL_UUID)); } if (!widgetsBundle.getTenantId().getId().equals(ModelConstants.NULL_UUID)) { - Tenant tenant = tenantDao.findById(tenantId, widgetsBundle.getTenantId().getId()); - if (tenant == null) { + if (!tenantService.tenantExists(widgetsBundle.getTenantId())) { throw new DataValidationException("Widgets bundle is referencing to non-existent tenant!"); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java index faea2cc34e..972d580cd7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsDao.java @@ -19,6 +19,8 @@ import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; +import java.util.UUID; + public interface AdminSettingsDao extends Dao { /** @@ -35,6 +37,10 @@ public interface AdminSettingsDao extends Dao { * @param key the key * @return the admin settings object */ - AdminSettings findByKey(TenantId tenantId, String key); + AdminSettings findByTenantIdAndKey(UUID tenantId, String key); + + boolean removeByTenantIdAndKey(UUID tenantId, String key); + + void removeByTenantId(UUID tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index bf4c71d296..28b4c97876 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; @@ -46,21 +47,40 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings findAdminSettingsByKey(TenantId tenantId, String key) { log.trace("Executing findAdminSettingsByKey [{}]", key); Validator.validateString(key, "Incorrect key " + key); - return adminSettingsDao.findByKey(tenantId, key); + return findAdminSettingsByTenantIdAndKey(TenantId.SYS_TENANT_ID, key); + } + + @Override + public AdminSettings findAdminSettingsByTenantIdAndKey(TenantId tenantId, String key) { + return adminSettingsDao.findByTenantIdAndKey(tenantId.getId(), key); } @Override public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if(adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { + if (adminSettings.getKey().equals("mail") && !adminSettings.getJsonValue().has("password")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); } } - + if (adminSettings.getTenantId() == null) { + adminSettings.setTenantId(TenantId.SYS_TENANT_ID); + } return adminSettingsDao.save(tenantId, adminSettings); } + @Override + public boolean deleteAdminSettingsByTenantIdAndKey(TenantId tenantId, String key) { + log.trace("Executing deleteAdminSettings, tenantId [{}], key [{}]", tenantId, key); + Validator.validateString(key, "Incorrect key " + key); + return adminSettingsDao.removeByTenantIdAndKey(tenantId.getId(), key); + } + + @Override + public void deleteAdminSettingsByTenantId(TenantId tenantId) { + adminSettingsDao.removeByTenantId(tenantId.getId()); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index 3c996ff6f5..ad6bb5ccec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -67,6 +67,14 @@ public abstract class JpaAbstractDao, D> return DaoUtil.getData(entity); } + @Override + @Transactional + public D saveAndFlush(TenantId tenantId, D domain) { + D d = save(tenantId, domain); + getRepository().flush(); + return d; + } + @Override public D findById(TenantId tenantId, UUID key) { log.debug("Get entity by key {}", key); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index 43d43a5f8e..0b409d9256 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -16,6 +16,9 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.EntityAlarmCompositeKey; import org.thingsboard.server.dao.model.sql.EntityAlarmEntity; @@ -28,5 +31,7 @@ public interface EntityAlarmRepository extends JpaRepository findAllByAlarmId(UUID alarmId); @Transactional - void deleteByEntityId(UUID id); + @Modifying + @Query("DELETE FROM EntityAlarmEntity e where e.entityId = :entityId") + void deleteByEntityId(@Param("entityId") UUID entityId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index cdf04f47c5..8e165898a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -188,4 +189,10 @@ public class JpaAlarmDao extends JpaAbstractDao implements A log.trace("[{}] Try to delete entity alarm records using [{}]", tenantId, entityId); entityAlarmRepository.deleteByEntityId(entityId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.ALARM; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java index 833fcc2de8..6d4667d8ec 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 5/21/2017. */ -public interface AssetRepository extends JpaRepository { +public interface AssetRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.AssetInfoEntity(a, c.title, c.additionalInfo) " + "FROM AssetEntity a " + @@ -143,4 +144,8 @@ public interface AssetRepository extends JpaRepository { Pageable pageable); Long countByTenantIdAndTypeIsNot(UUID tenantId, String type); + + @Query("SELECT externalId FROM AssetEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index c9752c5390..b18d6c5f51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetInfo; +import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -208,4 +209,31 @@ public class JpaAssetDao extends JpaAbstractSearchTextDao im public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantIdAndTypeIsNot(tenantId.getId(), TB_SERVICE_QUEUE); } + + @Override + public Asset findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(assetRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Asset findByTenantIdAndName(UUID tenantId, String name) { + return findAssetsByTenantIdAndName(tenantId, name).orElse(null); + } + + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findAssetsByTenantId(tenantId, pageLink); + } + + @Override + public AssetId getExternalIdByInternal(AssetId internalId) { + return Optional.ofNullable(assetRepository.getExternalIdById(internalId.getId())) + .map(AssetId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.ASSET; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java index 1a549436d0..4ec4a562c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/component/ComponentDescriptorRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.component; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.plugin.ComponentScope; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.dao.model.sql.ComponentDescriptorEntity; @@ -46,5 +48,8 @@ public interface ComponentDescriptorRepository extends JpaRepository { +public interface CustomerRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT c FROM CustomerEntity c WHERE c.tenantId = :tenantId " + "AND LOWER(c.searchText) LIKE LOWER(CONCAT('%', :textSearch, '%'))") @@ -38,4 +39,8 @@ public interface CustomerRepository extends JpaRepository CustomerEntity findByTenantIdAndTitle(UUID tenantId, String title); Long countByTenantId(UUID tenantId); + + @Query("SELECT externalId FROM CustomerEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java index 02311df61c..ab3a4b4f8d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/customer/JpaCustomerDao.java @@ -19,6 +19,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -68,4 +70,31 @@ public class JpaCustomerDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findCustomersByTenantId(tenantId, pageLink); + } + + @Override + public CustomerId getExternalIdByInternal(CustomerId internalId) { + return Optional.ofNullable(customerRepository.getExternalIdById(internalId.getId())) + .map(CustomerId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.CUSTOMER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java index 36518887d4..eaa247b96d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java @@ -15,15 +15,29 @@ */ package org.thingsboard.server.dao.sql.dashboard; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DashboardEntity; +import java.util.List; import java.util.UUID; /** * Created by Valerii Sosliuk on 5/6/2017. */ -public interface DashboardRepository extends JpaRepository { +public interface DashboardRepository extends JpaRepository, ExportableEntityRepository { Long countByTenantId(UUID tenantId); + + List findByTenantIdAndTitle(UUID tenantId, String title); + + Page findByTenantId(UUID tenantId, Pageable pageable); + + @Query("SELECT externalId FROM DashboardEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java index 98edcaa1fd..425c54c647 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/JpaDashboardDao.java @@ -19,11 +19,18 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dashboard.DashboardDao; import org.thingsboard.server.dao.model.sql.DashboardEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; +import java.util.List; +import java.util.Optional; import java.util.UUID; /** @@ -49,4 +56,31 @@ public class JpaDashboardDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return DaoUtil.toPageData(dashboardRepository.findByTenantId(tenantId, DaoUtil.toPageable(pageLink))); + } + + @Override + public DashboardId getExternalIdByInternal(DashboardId internalId) { + return Optional.ofNullable(dashboardRepository.getExternalIdById(internalId.getId())) + .map(DashboardId::new).orElse(null); + } + + @Override + public List findByTenantIdAndTitle(UUID tenantId, String title) { + return DaoUtil.convertDataList(dashboardRepository.findByTenantIdAndTitle(tenantId, title)); + } + + @Override + public EntityType getEntityType() { + return EntityType.DASHBOARD; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java index e61a60b5b6..fea0e4bfc0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java @@ -22,11 +22,12 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import java.util.UUID; -public interface DeviceProfileRepository extends JpaRepository { +public interface DeviceProfileRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.common.data.DeviceProfileInfo(d.id, d.name, d.image, d.defaultDashboardId, d.type, d.transportType) " + "FROM DeviceProfileEntity d " + @@ -66,4 +67,8 @@ public interface DeviceProfileRepository extends JpaRepository { +public interface DeviceRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.DeviceInfoEntity(d, c.title, c.additionalInfo, p.name) " + "FROM DeviceEntity d " + @@ -244,4 +245,8 @@ public interface DeviceRepository extends JpaRepository { "INNER JOIN DeviceProfileEntity p ON d.deviceProfileId = p.id " + "WHERE p.transportType = :transportType") Page findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType, Pageable pageable); + + @Query("SELECT externalId FROM DeviceEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 1d75690e50..8513e440b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageUtil; @@ -302,4 +303,31 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao Objects.toString(pageLink.getTextSearch(), ""), DaoUtil.toPageable(pageLink))); } + + @Override + public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { + return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); + } + + @Override + public Device findByTenantIdAndName(UUID tenantId, String name) { + return findDeviceByTenantIdAndName(tenantId, name).orElse(null); + } + + @Override + public PageData findByTenantId(UUID tenantId, PageLink pageLink) { + return findDevicesByTenantId(tenantId, pageLink); + } + + @Override + public DeviceId getExternalIdByInternal(DeviceId internalId) { + return Optional.ofNullable(deviceRepository.getExternalIdById(internalId.getId())) + .map(DeviceId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java index 80513d77c6..07a680493f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java @@ -23,6 +23,8 @@ import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -32,6 +34,7 @@ import org.thingsboard.server.dao.model.sql.DeviceProfileEntity; import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.Objects; +import java.util.Optional; import java.util.UUID; @Component @@ -109,4 +112,31 @@ public class JpaDeviceProfileDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findDeviceProfiles(TenantId.fromUUID(tenantId), pageLink); + } + + @Override + public DeviceProfileId getExternalIdByInternal(DeviceProfileId internalId) { + return Optional.ofNullable(deviceProfileRepository.getExternalIdById(internalId.getId())) + .map(DeviceProfileId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.DEVICE_PROFILE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java index 4be9c08849..802c9b99b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaEdgeDao.java @@ -193,4 +193,9 @@ public class JpaEdgeDao extends JpaAbstractSearchTextDao imple return list; } + @Override + public EntityType getEntityType() { + return EntityType.EDGE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java index 7a88aa3c5b..0999523c90 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/entityview/EntityViewRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.EntityViewEntity; import org.thingsboard.server.dao.model.sql.EntityViewInfoEntity; @@ -29,7 +30,7 @@ import java.util.UUID; /** * Created by Victor Basanets on 8/31/2017. */ -public interface EntityViewRepository extends JpaRepository { +public interface EntityViewRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT new org.thingsboard.server.dao.model.sql.EntityViewInfoEntity(e, c.title, c.additionalInfo) " + "FROM EntityViewEntity e " + @@ -139,4 +140,7 @@ public interface EntityViewRepository extends JpaRepository findByTenantId(UUID tenantId, PageLink pageLink) { + return findEntityViewsByTenantId(tenantId, pageLink); + } + + @Override + public EntityViewId getExternalIdByInternal(EntityViewId internalId) { + return Optional.ofNullable(entityViewRepository.getExternalIdById(internalId.getId())) + .map(EntityViewId::new).orElse(null); + } + + @Override + public EntityView findByTenantIdAndName(UUID tenantId, String name) { + return findEntityViewByTenantIdAndName(tenantId, name).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.ENTITY_VIEW; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java index 53c1318948..6af11346fd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/ota/JpaOtaPackageDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.sql.OtaPackageEntity; @@ -48,4 +49,10 @@ public class JpaOtaPackageDao extends JpaAbstractSearchTextDao(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.WIDGETS_BUNDLE, new HashSet<>(widgetEntityFields)); allowedEntityFieldMap.put(EntityType.API_USAGE_STATE, apiUsageStateEntityFields); + allowedEntityFieldMap.put(EntityType.DEVICE_PROFILE, Set.of(CREATED_TIME, NAME, TYPE)); entityFieldColumnMap.put(CREATED_TIME, ModelConstants.CREATED_TIME_PROPERTY); entityFieldColumnMap.put(ENTITY_TYPE, ModelConstants.ENTITY_TYPE_PROPERTY); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java deleted file mode 100644 index ec6df53ae2..0000000000 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/AbstractRelationInsertRepository.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.dao.sql.relation; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.jpa.repository.Modifying; -import org.thingsboard.server.dao.model.sql.RelationEntity; - -import javax.persistence.EntityManager; -import javax.persistence.PersistenceContext; -import javax.persistence.Query; - -@Slf4j -public abstract class AbstractRelationInsertRepository implements RelationInsertRepository { - - @PersistenceContext - protected EntityManager entityManager; - - protected Query getQuery(RelationEntity entity, String query) { - Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); - if (entity.getAdditionalInfo() == null) { - nativeQuery.setParameter("additionalInfo", null); - } else { - nativeQuery.setParameter("additionalInfo", entity.getAdditionalInfo().toString()); - } - return nativeQuery - .setParameter("fromId", entity.getFromId()) - .setParameter("fromType", entity.getFromType()) - .setParameter("toId", entity.getToId()) - .setParameter("toType", entity.getToType()) - .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) - .setParameter("relationType", entity.getRelationType()); - } - - @Modifying - protected abstract RelationEntity processSaveOrUpdate(RelationEntity entity); - -} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 71e36fcae1..422e0a8623 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -33,7 +33,11 @@ import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; /** * Created by Valerii Sosliuk on 5/29/2017. @@ -42,6 +46,12 @@ import java.util.List; @Component public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { + private static final List ALL_TYPE_GROUP_NAMES = new ArrayList<>(); + + static { + Arrays.stream(RelationTypeGroup.values()).map(RelationTypeGroup::name).forEach(ALL_TYPE_GROUP_NAMES::add); + } + @Autowired private RelationRepository relationRepository; @@ -57,6 +67,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByFrom(TenantId tenantId, EntityId from) { + return DaoUtil.convertDataList( + relationRepository.findAllByFromIdAndFromTypeAndRelationTypeGroupIn( + from.getId(), + from.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -76,6 +95,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByTo(TenantId tenantId, EntityId to) { + return DaoUtil.convertDataList( + relationRepository.findAllByToIdAndToTypeAndRelationTypeGroupIn( + to.getId(), + to.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -87,9 +115,14 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + public ListenableFuture checkRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { + return service.submit(() -> checkRelation(tenantId, from, to, relationType, typeGroup)); + } + + @Override + public boolean checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup) { RelationCompositeKey key = getRelationCompositeKey(from, to, relationType, typeGroup); - return service.submit(() -> relationRepository.existsById(key)); + return relationRepository.existsById(key); } @Override @@ -112,6 +145,12 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple return relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null; } + @Override + public void saveRelations(TenantId tenantId, Collection relations) { + List entities = relations.stream().map(RelationEntity::new).collect(Collectors.toList()); + relationInsertRepository.saveOrUpdate(entities); + } + @Override public ListenableFuture saveRelationAsync(TenantId tenantId, EntityRelation relation) { return service.submit(() -> relationInsertRepository.saveOrUpdate(new RelationEntity(relation)) != null); @@ -156,19 +195,21 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public boolean deleteOutboundRelations(TenantId tenantId, EntityId entity) { - boolean relationExistsBeforeDelete = false; + public void deleteOutboundRelations(TenantId tenantId, EntityId entity) { try { - relationExistsBeforeDelete = relationRepository - .findAllByFromIdAndFromType(entity.getId(), entity.getEntityType().name()) - .size() > 0; - if (relationExistsBeforeDelete) { - relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); - } + relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", entity, e); + } + } + + @Override + public void deleteInboundRelations(TenantId tenantId, EntityId entity) { + try { + relationRepository.deleteByToIdAndToTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), ALL_TYPE_GROUP_NAMES); } catch (ConcurrencyFailureException e) { log.debug("Concurrency exception while deleting relations [{}]", entity, e); } - return relationExistsBeforeDelete; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java index a07b11bcda..527a7f2bcd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationInsertRepository.java @@ -17,8 +17,12 @@ package org.thingsboard.server.dao.sql.relation; import org.thingsboard.server.dao.model.sql.RelationEntity; +import java.util.List; + public interface RelationInsertRepository { RelationEntity saveOrUpdate(RelationEntity entity); + void saveOrUpdate(List entities); + } \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index bddd344672..d71077c55e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.relation; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; @@ -35,6 +36,10 @@ public interface RelationRepository String fromType, String relationTypeGroup); + List findAllByFromIdAndFromTypeAndRelationTypeGroupIn(UUID fromId, + String fromType, + List relationTypeGroups); + List findAllByFromIdAndFromTypeAndRelationTypeAndRelationTypeGroup(UUID fromId, String fromType, String relationType, @@ -44,6 +49,10 @@ public interface RelationRepository String toType, String relationTypeGroup); + List findAllByToIdAndToTypeAndRelationTypeGroupIn(UUID toId, + String toType, + List relationTypeGroups); + List findAllByToIdAndToTypeAndRelationTypeAndRelationTypeGroup(UUID toId, String toType, String relationType, @@ -64,6 +73,13 @@ public interface RelationRepository void deleteById(RelationCompositeKey id); @Transactional - void deleteByFromIdAndFromType(UUID fromId, String fromType); + @Modifying + @Query("DELETE FROM RelationEntity r where r.fromId = :fromId and r.fromType = :fromType") + void deleteByFromIdAndFromType(@Param("fromId") UUID fromId, @Param("fromType") String fromType); + + @Transactional + @Modifying + @Query("DELETE FROM RelationEntity r where r.toId = :toId and r.toType = :toType and r.relationTypeGroup in :relationTypeGroups") + void deleteByToIdAndToTypeAndRelationTypeGroupIn(@Param("toId") UUID toId, @Param("toType") String toType, @Param("relationTypeGroups") List relationTypeGroups); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java index 23d24588b9..cbc4ca1e02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -15,25 +15,90 @@ */ package org.thingsboard.server.dao.sql.relation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.model.sql.RelationEntity; +import javax.persistence.EntityManager; +import javax.persistence.PersistenceContext; +import javax.persistence.Query; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + @Repository @Transactional -public class SqlRelationInsertRepository extends AbstractRelationInsertRepository implements RelationInsertRepository { +public class SqlRelationInsertRepository implements RelationInsertRepository { - private static final String INSERT_ON_CONFLICT_DO_UPDATE = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JPA = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + " VALUES (:fromId, :fromType, :toId, :toType, :relationTypeGroup, :relationType, :additionalInfo) " + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = :additionalInfo returning *"; + private static final String INSERT_ON_CONFLICT_DO_UPDATE_JDBC = "INSERT INTO relation (from_id, from_type, to_id, to_type, relation_type_group, relation_type, additional_info)" + + " VALUES (?, ?, ?, ?, ?, ?, ?) " + + "ON CONFLICT (from_id, from_type, relation_type_group, relation_type, to_id, to_type) DO UPDATE SET additional_info = ?"; + + + @PersistenceContext + protected EntityManager entityManager; + + @Autowired + protected JdbcTemplate jdbcTemplate; + + protected Query getQuery(RelationEntity entity, String query) { + Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); + if (entity.getAdditionalInfo() == null) { + nativeQuery.setParameter("additionalInfo", null); + } else { + nativeQuery.setParameter("additionalInfo", JacksonUtil.toString(entity.getAdditionalInfo())); + } + return nativeQuery + .setParameter("fromId", entity.getFromId()) + .setParameter("fromType", entity.getFromType()) + .setParameter("toId", entity.getToId()) + .setParameter("toType", entity.getToType()) + .setParameter("relationTypeGroup", entity.getRelationTypeGroup()) + .setParameter("relationType", entity.getRelationType()); + } + @Override public RelationEntity saveOrUpdate(RelationEntity entity) { - return processSaveOrUpdate(entity); + return (RelationEntity) getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE_JPA).getSingleResult(); } @Override - protected RelationEntity processSaveOrUpdate(RelationEntity entity) { - return (RelationEntity) getQuery(entity, INSERT_ON_CONFLICT_DO_UPDATE).getSingleResult(); + public void saveOrUpdate(List entities) { + jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + RelationEntity relation = entities.get(i); + ps.setObject(1, relation.getFromId()); + ps.setString(2, relation.getFromType()); + ps.setObject(3, relation.getToId()); + ps.setString(4, relation.getToType()); + + ps.setString(5, relation.getRelationTypeGroup()); + ps.setString(6, relation.getRelationType()); + + if (relation.getAdditionalInfo() == null) { + ps.setString(7, null); + ps.setString(8, null); + } else { + String json = JacksonUtil.toString(relation.getAdditionalInfo()); + ps.setString(7, json); + ps.setString(8, json); + } + } + + @Override + public int getBatchSize() { + return entities.size(); + } + }); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java index dbf60af096..5e4914f470 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.sql.resource; import lombok.extern.slf4j.Slf4j; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.id.TenantId; @@ -96,4 +97,10 @@ public class JpaTbResourceDao extends JpaAbstractSearchTextDao implements RpcDao public Long deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); } + + @Override + public EntityType getEntityType() { + return EntityType.RPC; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java index 3a7fc921e5..9ec2bab280 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleChainDao.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,6 +33,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.Collection; import java.util.Objects; +import java.util.Optional; import java.util.UUID; @Slf4j @@ -108,4 +111,25 @@ public class JpaRuleChainDao extends JpaAbstractSearchTextDao findByTenantId(UUID tenantId, PageLink pageLink) { + return findRuleChainsByTenantId(tenantId, pageLink); + } + + @Override + public RuleChainId getExternalIdByInternal(RuleChainId internalId) { + return Optional.ofNullable(ruleChainRepository.getExternalIdById(internalId.getId())) + .map(RuleChainId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_CHAIN; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index a0f84758c6..7fc1229cd3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -19,6 +19,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,6 +34,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Component @@ -63,4 +67,20 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao findByExternalIds(RuleChainId ruleChainId, List externalIds) { + return DaoUtil.convertDataList(ruleNodeRepository.findRuleNodesByRuleChainIdAndExternalIdIn(ruleChainId.getId(), + externalIds.stream().map(RuleNodeId::getId).collect(Collectors.toList()))); + } + + @Override + public void deleteByIdIn(List ruleNodeIds) { + ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); + } + + @Override + public EntityType getEntityType() { + return EntityType.RULE_NODE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java index fe9be8dcc7..7c4b4b86f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleChainRepository.java @@ -21,12 +21,13 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.RuleChainEntity; import java.util.List; import java.util.UUID; -public interface RuleChainRepository extends JpaRepository { +public interface RuleChainRepository extends JpaRepository, ExportableEntityRepository { @Query("SELECT rc FROM RuleChainEntity rc WHERE rc.tenantId = :tenantId " + "AND LOWER(rc.searchText) LIKE LOWER(CONCAT('%', :searchText, '%'))") @@ -66,4 +67,7 @@ public interface RuleChainRepository extends JpaRepository findByTenantIdAndTypeAndName(UUID tenantId, RuleChainType type, String name); + @Query("SELECT externalId FROM RuleChainEntity WHERE id = :id") + UUID getExternalIdById(@Param("id") UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index f584ff2b02..e6a08b910a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -18,8 +18,10 @@ package org.thingsboard.server.dao.sql.rule; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.RuleNodeEntity; import java.util.List; @@ -31,12 +33,19 @@ public interface RuleNodeRepository extends JpaRepository "(select id from RuleChainEntity rc WHERE rc.tenantId = :tenantId) " + "AND r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") List findRuleNodesByTenantIdAndType(@Param("tenantId") UUID tenantId, - @Param("ruleType") String ruleType, - @Param("searchText") String searchText); + @Param("ruleType") String ruleType, + @Param("searchText") String searchText); @Query("SELECT r FROM RuleNodeEntity r WHERE r.type = :ruleType AND LOWER(r.configuration) LIKE LOWER(CONCAT('%', :searchText, '%')) ") Page findAllRuleNodesByType(@Param("ruleType") String ruleType, @Param("searchText") String searchText, Pageable pageable); + List findRuleNodesByRuleChainIdAndExternalIdIn(UUID ruleChainId, List externalIds); + + @Transactional + @Modifying + @Query("DELETE FROM RuleNodeEntity e where e.id in :ids") + void deleteByIdIn(@Param("ids") List ids); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java index 58423645b1..e607517f52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/AdminSettingsRepository.java @@ -25,5 +25,12 @@ import java.util.UUID; */ public interface AdminSettingsRepository extends JpaRepository { - AdminSettingsEntity findByKey(String key); + AdminSettingsEntity findByTenantIdAndKey(UUID tenantId, String key); + + void deleteByTenantIdAndKey(UUID tenantId, String key); + + void deleteByTenantId(UUID tenantId); + + boolean existsByTenantIdAndKey(UUID tenantId, String key); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java index 438274e3cc..bd815bc410 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/settings/JpaAdminSettingsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.DaoUtil; @@ -46,7 +47,24 @@ public class JpaAdminSettingsDao extends JpaAbstractDao return DaoUtil.pageToPageData(tenantRepository.findTenantsIds(DaoUtil.toPageable(pageLink))).mapData(TenantId::fromUUID); } + @Override + public EntityType getEntityType() { + return EntityType.TENANT; + } + @Override public List findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId) { return tenantRepository.findTenantIdsByTenantProfileId(tenantProfileId.getId()).stream() diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java index 7b060e0b43..8c123b12fb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/usagerecord/ApiUsageStateRepository.java @@ -42,5 +42,6 @@ public interface ApiUsageStateRepository extends JpaRepository imple public Long countByTenantId(TenantId tenantId) { return userRepository.countByTenantId(tenantId.getId()); } + + @Override + public EntityType getEntityType() { + return EntityType.USER; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java index 38642a0161..c5a8763209 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserAuthSettingsRepository.java @@ -16,6 +16,9 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.UserAuthSettingsEntity; @@ -28,6 +31,8 @@ public interface UserAuthSettingsRepository extends JpaRepository findByTenantId(UUID tenantId, PageLink pageLink) { + return findTenantWidgetsBundlesByTenantId(tenantId, pageLink); + } + + @Override + public WidgetsBundleId getExternalIdByInternal(WidgetsBundleId internalId) { + return Optional.ofNullable(widgetsBundleRepository.getExternalIdById(internalId.getId())) + .map(WidgetsBundleId::new).orElse(null); + } + + @Override + public EntityType getEntityType() { + return EntityType.WIDGETS_BUNDLE; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java index 0de8d0220b..188b42f646 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetsBundleRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.dao.ExportableEntityRepository; import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity; import java.util.UUID; @@ -27,7 +28,7 @@ import java.util.UUID; /** * Created by Valerii Sosliuk on 4/23/2017. */ -public interface WidgetsBundleRepository extends JpaRepository { +public interface WidgetsBundleRepository extends JpaRepository, ExportableEntityRepository { WidgetsBundleEntity findWidgetsBundleByTenantIdAndAlias(UUID tenantId, String alias); @@ -49,4 +50,10 @@ public interface WidgetsBundleRepository extends JpaRepository>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + query.getInterval(); + long endPeriod = query.getEndTs(); + long startPeriod = query.getStartTs(); + long step = query.getInterval(); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); long ts = startTs + (endTs - startTs) / 2; - futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); - stepTs = endTs; + ListenableFuture> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()); + futures.add(aggregateTsKvEntry); + startPeriod = endTs; } return getTskvEntriesFuture(Futures.allAsList(futures)); } @@ -148,7 +152,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); } - private ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { + ListenableFuture> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { List> entitiesFutures = new ArrayList<>(); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); return Futures.transform(setFutures(entitiesFutures), entity -> { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java index 6009f9babe..aa0d9b1c47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java @@ -91,7 +91,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries .stream() .map(query -> findAllAsync(tenantId, entityId, query)) .collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java index de97976714..31270bacc7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java @@ -26,4 +26,4 @@ import java.util.List; public interface AggregationTimeseriesDao { ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); -} +} \ No newline at end of file diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index 56fe4f0f6f..82e54168bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; @@ -39,9 +40,7 @@ import java.util.stream.Collectors; public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { private final ConcurrentMap tsKvDictionaryMap = new ConcurrentHashMap<>(); - protected static final ReentrantLock tsCreationLock = new ReentrantLock(); - @Autowired protected TsKvDictionaryRepository dictionaryRepository; @@ -61,7 +60,7 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); - } catch (ConstraintViolationException e) { + } catch (DataIntegrityViolationException | ConstraintViolationException e) { tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); @@ -96,4 +95,5 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni } }, service); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantCacheKey.java new file mode 100644 index 0000000000..8a0af86faa --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantCacheKey.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; + +import java.io.Serializable; + +@Data +@AllArgsConstructor +public class TenantCacheKey implements Serializable { + + private static final long serialVersionUID = -121787454251592384L; + + private final TenantId tenantId; + private final TenantCacheKeyPrefix keyPrefix; + + public static TenantCacheKey fromId(TenantId tenantId) { + return new TenantCacheKey(tenantId, TenantCacheKeyPrefix.TENANT); + } + + public static TenantCacheKey fromIdExists(TenantId tenantId) { + return new TenantCacheKey(tenantId, TenantCacheKeyPrefix.EXISTS); + } + + public enum TenantCacheKeyPrefix { + TENANT, EXISTS + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantCaffeineCache.java new file mode 100644 index 0000000000..e6ae0d930f --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("TenantCache") +public class TenantCaffeineCache extends CaffeineTbTransactionalCache { + + public TenantCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.TENANTS_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantEvictEvent.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantEvictEvent.java new file mode 100644 index 0000000000..2bd28ae511 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantEvictEvent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +public class TenantEvictEvent { + private final TenantId tenantId; + // for exists tenant cache + private final boolean isExistsTenant; +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsCaffeineCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsCaffeineCache.java new file mode 100644 index 0000000000..c7b25a5059 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsCaffeineCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CaffeineTbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true) +@Service("TenantExistsCache") +public class TenantExistsCaffeineCache extends CaffeineTbTransactionalCache { + + public TenantExistsCaffeineCache(CacheManager cacheManager) { + super(cacheManager, CacheConstants.TENANTS_CACHE); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java new file mode 100644 index 0000000000..4f938bf690 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("TenantExistsCache") +public class TenantExistsRedisCache extends RedisTbTransactionalCache { + + public TenantExistsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java new file mode 100644 index 0000000000..195e781755 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.tenant; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; +import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Tenant; + +@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") +@Service("TenantCache") +public class TenantRedisCache extends RedisTbTransactionalCache { + + public TenantRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { + super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 0590286714..a6fefd3ead 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -18,7 +18,11 @@ package org.thingsboard.server.dao.tenant; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; @@ -31,8 +35,7 @@ import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.entity.AbstractEntityService; -import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.resource.ResourceService; @@ -41,6 +44,7 @@ import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.service.Validator; +import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetsBundleService; @@ -51,7 +55,7 @@ import static org.thingsboard.server.dao.service.Validator.validateId; @Service @Slf4j -public class TenantServiceImpl extends AbstractEntityService implements TenantService { +public class TenantServiceImpl extends AbstractCachedEntityService implements TenantService { private static final String DEFAULT_TENANT_REGION = "Global"; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; @@ -63,6 +67,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe private TenantProfileService tenantProfileService; @Autowired + @Lazy private UserService userService; @Autowired @@ -77,12 +82,10 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private DeviceProfileService deviceProfileService; + @Lazy @Autowired private ApiUsageStateService apiUsageStateService; - @Autowired - private EntityViewService entityViewService; - @Autowired private WidgetsBundleService widgetsBundleService; @@ -104,14 +107,33 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Autowired private DataValidator tenantValidator; + @Lazy @Autowired private QueueService queueService; + @Autowired + private AdminSettingsService adminSettingsService; + + @Autowired + protected TbTransactionalCache existsTenantCache; + + @TransactionalEventListener(classes = TenantEvictEvent.class) + @Override + public void handleEvictEvent(TenantEvictEvent event) { + TenantId tenantId = event.getTenantId(); + cache.evict(TenantCacheKey.fromId(tenantId)); + if (event.isExistsTenant()) { + existsTenantCache.evict(TenantCacheKey.fromIdExists(tenantId)); + } + } + @Override public Tenant findTenantById(TenantId tenantId) { log.trace("Executing findTenantById [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); - return tenantDao.findById(tenantId, tenantId.getId()); + + return cache.getAndPutInTransaction(TenantCacheKey.fromId(tenantId), + () -> tenantDao.findById(tenantId, tenantId.getId()), true); } @Override @@ -123,12 +145,13 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe @Override public ListenableFuture findTenantByIdAsync(TenantId callerId, TenantId tenantId) { - log.trace("Executing TenantIdAsync [{}]", tenantId); + log.trace("Executing findTenantByIdAsync [{}]", tenantId); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); return tenantDao.findByIdAsync(callerId, tenantId.getId()); } @Override + @Transactional public Tenant saveTenant(Tenant tenant) { log.trace("Executing saveTenant [{}]", tenant); tenant.setRegion(DEFAULT_TENANT_REGION); @@ -138,6 +161,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } tenantValidator.validate(tenant, Tenant::getId); Tenant savedTenant = tenantDao.save(tenant.getId(), tenant); + publishEvictEvent(new TenantEvictEvent(savedTenant.getId(), false)); if (tenant.getId() == null) { deviceProfileService.createDefaultDeviceProfile(savedTenant.getId()); apiUsageStateService.createDefaultApiUsageState(savedTenant.getId(), null); @@ -146,6 +170,7 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe } @Override + @Transactional(timeout = 60 * 60) public void deleteTenant(TenantId tenantId) { log.trace("Executing deleteTenant [{}]", tenantId); Validator.validateId(tenantId, INCORRECT_TENANT_ID + tenantId); @@ -164,7 +189,9 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe otaPackageService.deleteOtaPackagesByTenantId(tenantId); rpcService.deleteAllRpcByTenantId(tenantId); queueService.deleteQueuesByTenantId(tenantId); + adminSettingsService.deleteAdminSettingsByTenantId(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); + publishEvictEvent(new TenantEvictEvent(tenantId, true)); deleteEntityRelations(tenantId, tenantId); } @@ -194,17 +221,29 @@ public class TenantServiceImpl extends AbstractEntityService implements TenantSe tenantsRemover.removeEntities(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID); } - private PaginatedRemover tenantsRemover = - new PaginatedRemover<>() { + @Override + public PageData findTenantsIds(PageLink pageLink) { + log.trace("Executing findTenantsIds"); + Validator.validatePageLink(pageLink); + return tenantDao.findTenantsIds(pageLink); + } + + @Override + public boolean tenantExists(TenantId tenantId) { + return existsTenantCache.getAndPutInTransaction(TenantCacheKey.fromIdExists(tenantId), + () -> tenantDao.existsById(tenantId, tenantId.getId()), false); + } - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return tenantDao.findTenants(tenantId, pageLink); - } + private PaginatedRemover tenantsRemover = new PaginatedRemover<>() { - @Override - protected void removeEntity(TenantId tenantId, Tenant entity) { - deleteTenant(TenantId.fromUUID(entity.getUuidId())); - } - }; + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return tenantDao.findTenants(tenantId, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, Tenant entity) { + deleteTenant(TenantId.fromUUID(entity.getUuidId())); + } + }; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 794c4d52d5..bb3930b99c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -141,7 +141,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD @Override public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { List>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); - return Futures.transform(Futures.allAsList(futures), new Function>, List>() { + return Futures.transform(Futures.allAsList(futures), new Function<>() { @Nullable @Override public List apply(@Nullable List> results) { @@ -270,18 +270,20 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD if (query.getAggregation() == Aggregation.NONE) { return findAllAsyncWithLimit(tenantId, entityId, query); } else { + long startPeriod = query.getStartTs(); + long endPeriod = query.getEndTs(); long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); - long stepTs = query.getStartTs(); List>> futures = new ArrayList<>(); - while (stepTs < query.getEndTs()) { - long startTs = stepTs; - long endTs = stepTs + step; - ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder()); + while (startPeriod <= endPeriod) { + long startTs = startPeriod; + long endTs = Math.min(startPeriod + step, endPeriod + 1); + long ts = endTs - startTs; + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder()); futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); - stepTs = endTs; + startPeriod = endTs; } ListenableFuture>> future = Futures.allAsList(futures); - return Futures.transform(future, new Function>, List>() { + return Futures.transform(future, new Function<>() { @Nullable @Override public List apply(@Nullable List> input) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index db52629d8d..f2bd8e47c1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -35,8 +35,8 @@ import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.service.DataValidator; -import org.thingsboard.server.dao.tenant.TenantDao; import org.thingsboard.server.dao.tenant.TenantProfileDao; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.ArrayList; @@ -52,16 +52,16 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A private final ApiUsageStateDao apiUsageStateDao; private final TenantProfileDao tenantProfileDao; - private final TenantDao tenantDao; + private final TenantService tenantService; private final TimeseriesService tsService; private final DataValidator apiUsageStateValidator; public ApiUsageStateServiceImpl(ApiUsageStateDao apiUsageStateDao, TenantProfileDao tenantProfileDao, - TenantDao tenantDao, @Lazy TimeseriesService tsService, + TenantService tenantService, @Lazy TimeseriesService tsService, DataValidator apiUsageStateValidator) { this.apiUsageStateDao = apiUsageStateDao; this.tenantProfileDao = tenantProfileDao; - this.tenantDao = tenantDao; + this.tenantService = tenantService; this.tsService = tsService; this.apiUsageStateValidator = apiUsageStateValidator; } @@ -118,7 +118,7 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A if (entityId.getEntityType() == EntityType.TENANT && !entityId.equals(TenantId.SYS_TENANT_ID)) { tenantId = (TenantId) entityId; - Tenant tenant = tenantDao.findById(tenantId, tenantId.getId()); + Tenant tenant = tenantService.findTenantById(tenantId); TenantProfile tenantProfile = tenantProfileDao.findById(tenantId, tenant.getTenantProfileId().getId()); TenantProfileConfiguration configuration = tenantProfile.getProfileData().getConfiguration(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 24b5dc837e..7c8a8ca27b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.ListenableFuture; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Value; @@ -50,6 +51,7 @@ import static org.thingsboard.server.dao.service.Validator.validateString; @Service @Slf4j +@RequiredArgsConstructor public class UserServiceImpl extends AbstractEntityService implements UserService { public static final String USER_PASSWORD_HISTORY = "userPasswordHistory"; @@ -73,20 +75,6 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic private final DataValidator userCredentialsValidator; private final ApplicationEventPublisher eventPublisher; - public UserServiceImpl(UserDao userDao, - UserCredentialsDao userCredentialsDao, - UserAuthSettingsDao userAuthSettingsDao, - DataValidator userValidator, - DataValidator userCredentialsValidator, - ApplicationEventPublisher eventPublisher) { - this.userDao = userDao; - this.userCredentialsDao = userCredentialsDao; - this.userAuthSettingsDao = userAuthSettingsDao; - this.userValidator = userValidator; - this.userCredentialsValidator = userCredentialsValidator; - this.eventPublisher = eventPublisher; - } - @Override public User findUserByEmail(TenantId tenantId, String email) { log.trace("Executing findUserByEmail [{}]", email); diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java index 2850349470..4c45239921 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java @@ -116,8 +116,13 @@ public abstract class AbstractBufferedRateExecutor { +public interface WidgetsBundleDao extends Dao, ExportableEntityDao { /** * Save or update widgets bundle object diff --git a/dao/src/main/resources/sql/schema-entities-idx.sql b/dao/src/main/resources/sql/schema-entities-idx.sql index b74d935c12..d4537a700d 100644 --- a/dao/src/main/resources/sql/schema-entities-idx.sql +++ b/dao/src/main/resources/sql/schema-entities-idx.sql @@ -51,3 +51,23 @@ CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribu CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); + +CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_node_external_id ON rule_node(rule_chain_id, external_id); + +CREATE INDEX IF NOT EXISTS idx_rule_node_type ON rule_node(type); \ No newline at end of file diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index ad9835af33..953276cbee 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -34,6 +34,7 @@ call insert_tb_schema_settings(); CREATE TABLE IF NOT EXISTS admin_settings ( id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, + tenant_id uuid NOT NULL, created_time bigint NOT NULL, json_value varchar, key varchar(255) @@ -82,6 +83,7 @@ CREATE TABLE IF NOT EXISTS asset ( search_text varchar(255), tenant_id uuid, type varchar(255), + external_id uuid, CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) ); @@ -141,7 +143,8 @@ CREATE TABLE IF NOT EXISTS customer ( state varchar(255), tenant_id uuid, title varchar(255), - zip varchar(255) + zip varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS dashboard ( @@ -154,7 +157,8 @@ CREATE TABLE IF NOT EXISTS dashboard ( title varchar(255), mobile_hide boolean DEFAULT false, mobile_order int, - image varchar(1000000) + image varchar(1000000), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_chain ( @@ -168,7 +172,8 @@ CREATE TABLE IF NOT EXISTS rule_chain ( root boolean, debug_mode boolean, search_text varchar(255), - tenant_id uuid + tenant_id uuid, + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node ( @@ -180,7 +185,8 @@ CREATE TABLE IF NOT EXISTS rule_node ( type varchar(255), name varchar(255), debug_mode boolean, - search_text varchar(255) + search_text varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS rule_node_state ( @@ -249,6 +255,7 @@ CREATE TABLE IF NOT EXISTS device_profile ( default_dashboard_id uuid, default_queue_id uuid, provision_device_key varchar, + external_id uuid, CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), @@ -293,6 +300,7 @@ CREATE TABLE IF NOT EXISTS device ( tenant_id uuid, firmware_id uuid, software_id uuid, + external_id uuid, CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), @@ -416,7 +424,8 @@ CREATE TABLE IF NOT EXISTS widgets_bundle ( tenant_id uuid, title varchar(255), image varchar(1000000), - description varchar(255) + description varchar(255), + external_id uuid ); CREATE TABLE IF NOT EXISTS entity_view ( @@ -432,7 +441,8 @@ CREATE TABLE IF NOT EXISTS entity_view ( start_ts bigint, end_ts bigint, search_text varchar(255), - additional_info varchar + additional_info varchar, + external_id uuid ); CREATE TABLE IF NOT EXISTS ts_kv_latest diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index ef163b2a06..964e6229f1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -65,6 +65,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.ResourceService; +import org.thingsboard.server.dao.rpc.RpcService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.dao.tenant.TenantProfileService; @@ -170,6 +171,9 @@ public abstract class AbstractServiceTest { @Autowired protected OtaPackageService otaPackageService; + @Autowired + protected RpcService rpcService; + @Autowired protected QueueService queueService; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeServiceTest.java index 0bbbf82452..093bd386a7 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEdgeServiceTest.java @@ -16,11 +16,13 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.Tenant; @@ -29,9 +31,14 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -602,4 +609,55 @@ public abstract class BaseEdgeServiceTest extends AbstractServiceTest { edgeService.deleteEdge(tenantId, savedEdge.getId()); } -} \ No newline at end of file + @Test + public void testFindMissingToRelatedRuleChains() { + Edge edge = constructEdge("My edge", "default"); + Edge savedEdge = edgeService.saveEdge(edge); + + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName("Rule Chain #1"); + ruleChain.setType(RuleChainType.EDGE); + RuleChain ruleChain1 = ruleChainService.saveRuleChain(ruleChain); + + ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName("Rule Chain #2"); + ruleChain.setType(RuleChainType.EDGE); + RuleChain ruleChain2 = ruleChainService.saveRuleChain(ruleChain); + + ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName("Rule Chain #3"); + ruleChain.setType(RuleChainType.EDGE); + RuleChain ruleChain3 = ruleChainService.saveRuleChain(ruleChain); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Input rule node 1"); + ruleNode1.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + ObjectNode configuration = JacksonUtil.OBJECT_MAPPER.createObjectNode(); + configuration.put("ruleChainId", ruleChain1.getUuidId().toString()); + ruleNode1.setConfiguration(configuration); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Input rule node 2"); + ruleNode2.setType("org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + configuration = JacksonUtil.OBJECT_MAPPER.createObjectNode(); + configuration.put("ruleChainId", ruleChain2.getUuidId().toString()); + ruleNode2.setConfiguration(configuration); + + RuleChainMetaData ruleChainMetaData3 = new RuleChainMetaData(); + ruleChainMetaData3.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + ruleChainMetaData3.setFirstNodeIndex(0); + ruleChainMetaData3.setRuleChainId(ruleChain3.getId()); + ruleChainService.saveRuleChainMetaData(tenantId, ruleChainMetaData3); + + ruleChainService.assignRuleChainToEdge(tenantId, ruleChain3.getId(), savedEdge.getId()); + + String missingToRelatedRuleChains = edgeService.findMissingToRelatedRuleChains(tenantId, + savedEdge.getId(), + "org.thingsboard.rule.engine.flow.TbRuleChainInputNode"); + Assert.assertEquals("{\"Rule Chain #3\":[\"Rule Chain #1\",\"Rule Chain #2\"]}", missingToRelatedRuleChains); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 6f088c79f6..ade4ce8c28 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -97,26 +97,26 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(0, otaPackageService.sumDataSizeByTenantId(tenantId)); - createFirmware(tenantId, "1"); + createAndSaveFirmware(tenantId, "1"); Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); thrown.expect(DataValidationException.class); thrown.expectMessage(String.format("Failed to create the ota package, files size limit is exhausted %d bytes!", DATA_SIZE)); - createFirmware(tenantId, "2"); + createAndSaveFirmware(tenantId, "2"); } @Test public void sumDataSizeByTenantId() { Assert.assertEquals(0, otaPackageService.sumDataSizeByTenantId(tenantId)); - createFirmware(tenantId, "0.1"); + createAndSaveFirmware(tenantId, "0.1"); Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; List packages = new ArrayList<>(maxSumDataSize); for (int i = 2; i <= maxSumDataSize; i++) { - packages.add(createFirmware(tenantId, "0." + i)); + packages.add(createAndSaveFirmware(tenantId, "0." + i)); Assert.assertEquals(i, otaPackageService.sumDataSizeByTenantId(tenantId)); } @@ -419,15 +419,15 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testSaveFirmwareWithExistingTitleAndVersion() { - createFirmware(tenantId, VERSION); + createAndSaveFirmware(tenantId, VERSION); thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage with such title and version already exists!"); - createFirmware(tenantId, VERSION); + createAndSaveFirmware(tenantId, VERSION); } @Test public void testDeleteFirmwareWithReferenceByDevice() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); Device device = new Device(); device.setTenantId(tenantId); @@ -448,7 +448,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testUpdateDeviceProfileId() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); try { thrown.expect(DataValidationException.class); @@ -493,7 +493,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testFindFirmwareById() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertNotNull(foundFirmware); @@ -519,7 +519,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testDeleteFirmware() { - OtaPackage savedFirmware = createFirmware(tenantId, VERSION); + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertNotNull(foundFirmware); @@ -532,7 +532,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public void testFindTenantFirmwaresByTenantId() { List firmwares = new ArrayList<>(); for (int i = 0; i < 165; i++) { - OtaPackageInfo info = new OtaPackageInfo(createFirmware(tenantId, VERSION + i)); + OtaPackageInfo info = new OtaPackageInfo(createAndSaveFirmware(tenantId, VERSION + i)); info.setHasData(true); firmwares.add(info); } @@ -579,7 +579,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public void testFindTenantFirmwaresByTenantIdAndHasData() { List firmwares = new ArrayList<>(); for (int i = 0; i < 165; i++) { - firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createFirmware(tenantId, VERSION + i)))); + firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i)))); } OtaPackageInfo firmwareWithUrl = new OtaPackageInfo(); @@ -695,7 +695,15 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { otaPackageService.saveOtaPackageInfo(firmwareInfo, true); } - private OtaPackage createFirmware(TenantId tenantId, String version) { + private OtaPackage createAndSaveFirmware(TenantId tenantId, String version) { + return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId)); + } + + public static OtaPackage createFirmware( + TenantId tenantId, + String version, + DeviceProfileId deviceProfileId + ) { OtaPackage firmware = new OtaPackage(); firmware.setTenantId(tenantId); firmware.setDeviceProfileId(deviceProfileId); @@ -708,6 +716,6 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setChecksum(CHECKSUM); firmware.setData(DATA); firmware.setDataSize(DATA_SIZE); - return otaPackageService.saveOtaPackage(firmware); + return firmware; } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java index 1f167976e5..75ac95d470 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseRelationServiceTest.java @@ -57,13 +57,13 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertTrue(saveRelation(relation)); - Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, parentId, "NOT_EXISTING_TYPE", RelationTypeGroup.COMMON)); } @Test @@ -80,9 +80,9 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { Assert.assertTrue(relationService.deleteRelationAsync(SYSTEM_TENANT_ID, relationA).get()); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertTrue(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); Assert.assertTrue(relationService.deleteRelationAsync(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); } @@ -116,11 +116,11 @@ public abstract class BaseRelationServiceTest extends AbstractServiceTest { saveRelation(relationA); saveRelation(relationB); - Assert.assertNull(relationService.deleteEntityRelationsAsync(SYSTEM_TENANT_ID, childId).get()); + relationService.deleteEntityRelations(SYSTEM_TENANT_ID, childId); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, parentId, childId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); - Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON).get()); + Assert.assertFalse(relationService.checkRelation(SYSTEM_TENANT_ID, childId, subChildId, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON)); } @Test diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 6017883514..d893a1ba40 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -17,26 +17,77 @@ package org.thingsboard.server.dao.service; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.TbCacheValueWrapper; +import org.thingsboard.server.cache.TbTransactionalCache; +import org.thingsboard.server.common.data.CacheConstants; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.rpc.Rpc; +import org.thingsboard.server.common.data.rpc.RpcStatus; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.tenant.TenantCacheKey; +import org.thingsboard.server.dao.tenant.TenantDao; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; +import java.util.Objects; +import java.util.Set; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.assertj.core.api.Assertions.assertThat; public abstract class BaseTenantServiceTest extends AbstractServiceTest { - + private IdComparator idComparator = new IdComparator<>(); + @SpyBean + protected TenantDao tenantDao; + + @Autowired + protected TbTransactionalCache cache; + + @Autowired + protected TbTransactionalCache existsTenantCache; + @Test public void testSaveTenant() { Tenant tenant = new Tenant(); @@ -103,7 +154,6 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { @Test public void testFindTenants() { - List tenants = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = tenantService.findTenants(pageLink); @@ -275,4 +325,382 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenant.setTenantProfileId(isolatedTenantProfile.getId()); tenantService.saveTenant(tenant); } + + @Test + public void testGettingTenantAddingItToCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Mockito.reset(tenantDao); + + verify(tenantDao, Mockito.times(0)).findById(any(), any()); + tenantService.findTenantById(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + var cachedTenant = cache.get(TenantCacheKey.fromId(savedTenant.getId())); + Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertEquals(savedTenant, cachedTenant.get()); + + for (int i = 0; i < 100; i++) { + tenantService.findTenantById(savedTenant.getId()); + } + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testExistsTenantAddingResultToCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + Mockito.reset(tenantDao); + //fromIdExists invoked from device profile validator + existsTenantCache.evict(TenantCacheKey.fromIdExists(savedTenant.getTenantId())); + + verify(tenantDao, Mockito.times(0)).existsById(any(), any()); + tenantService.tenantExists(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + var isExists = existsTenantCache.get(TenantCacheKey.fromIdExists(savedTenant.getId())); + Assert.assertNotNull("Getting an existing Tenant doesn't add it to the cache!", isExists); + + for (int i = 0; i < 100; i++) { + tenantService.tenantExists(savedTenant.getId()); + } + verify(tenantDao, Mockito.times(1)).existsById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testUpdatingExistingTenantEvictCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + tenantService.findTenantById(savedTenant.getId()); + + var cachedTenant = cache.get(TenantCacheKey.fromId(savedTenant.getId())); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertEquals(savedTenant, cachedTenant.get()); + + savedTenant.setTitle("My new tenant"); + savedTenant = tenantService.saveTenant(savedTenant); + + Mockito.reset(tenantDao); + + cachedTenant = cache.get(TenantCacheKey.fromId(savedTenant.getId())); + Assert.assertNull("Updating a Tenant doesn't evict the cache!", cachedTenant); + + verify(tenantDao, Mockito.times(0)).findById(any(), any()); + tenantService.findTenantById(savedTenant.getId()); + verify(tenantDao, Mockito.times(1)).findById(eq(savedTenant.getId()), eq(savedTenant.getId().getId())); + + tenantService.deleteTenant(savedTenant.getId()); + } + + @Test + public void testRemovingTenantEvictCache() { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + Tenant savedTenant = tenantService.saveTenant(tenant); + + tenantService.findTenantById(savedTenant.getId()); + tenantService.tenantExists(savedTenant.getId()); + + var cachedTenant = + cache.get(TenantCacheKey.fromId(savedTenant.getId())); + var cachedExists = + existsTenantCache.get(TenantCacheKey.fromIdExists(savedTenant.getId())); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedTenant); + Assert.assertNotNull("Saving a Tenant doesn't add it to the cache!", cachedExists); + + tenantService.deleteTenant(savedTenant.getId()); + cachedTenant = + cache.get(TenantCacheKey.fromId(savedTenant.getId())); + cachedExists = + existsTenantCache.get(TenantCacheKey.fromIdExists(savedTenant.getId())); + + + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedTenant); + Assert.assertNull("Removing a Tenant doesn't evict the cache!", cachedExists); + } + + @Test + public void testDeleteTenantDeletingAllRelatedEntities() throws Exception { + TenantProfile profile = createAndSaveTenantProfile(); + Tenant tenant = createAndSaveTenant(profile); + User user = createAndSaveUserFor(tenant); + Customer customer = createAndSaveCustomerFor(tenant); + WidgetsBundle widgetsBundle = createAndSaveWidgetBundleFor(tenant); + DeviceProfile deviceProfile = createAndSaveDeviceProfileWithProfileDataFor(tenant); + Device device = createAndSaveDeviceFor(tenant, customer, deviceProfile); + EntityView entityView = createAndSaveEntityViewFor(tenant, customer, device); + Asset asset = createAndSaveAssetFor(tenant, customer); + Dashboard dashboard = createAndSaveDashboardFor(tenant, customer); + RuleChain ruleChain = createAndSaveRuleChainFor(tenant); + Edge edge = createAndSaveEdgeFor(tenant); + OtaPackage otaPackage = createAndSaveOtaPackageFor(tenant, deviceProfile); + TbResource resource = createAndSaveResourceFor(tenant); + Rpc rpc = createAndSaveRpcFor(tenant, device); + + tenantService.deleteTenant(tenant.getId()); + + Assert.assertNull(tenantService.findTenantById(tenant.getId())); + assertCustomerIsDeleted(tenant, customer); + assertWidgetsBundleIsDeleted(tenant, widgetsBundle); + assertEntityViewIsDeleted(tenant, device, entityView); + assertAssetIsDeleted(tenant, asset); + assertDeviceIsDeleted(tenant, device); + assertDeviceProfileIsDeleted(tenant, deviceProfile); + assertDashboardIsDeleted(tenant, dashboard); + assertEdgeIsDeleted(tenant, edge); + assertTenantAdminIsDeleted(tenant); + assertUserIsDeleted(tenant, user); + Assert.assertNull(ruleChainService.findRuleChainById(tenant.getId(), ruleChain.getId())); + Assert.assertNull(apiUsageStateService.findTenantApiUsageState(tenant.getId())); + assertResourceIsDeleted(tenant, resource); + assertOtaPackageIsDeleted(tenant, otaPackage); + Assert.assertNull(rpcService.findById(tenant.getId(), rpc.getId())); + + tenantProfileService.deleteTenantProfile(TenantId.SYS_TENANT_ID, profile.getId()); + } + + private void assertOtaPackageIsDeleted(Tenant tenant, OtaPackage otaPackage) { + assertThat(otaPackageService.findOtaPackageById(tenant.getId(), otaPackage.getId())) + .as("otaPackage").isNull(); + PageLink pageLinkOta = new PageLink(1); + PageData pageDataOta = otaPackageService.findTenantOtaPackagesByTenantId(tenant.getId(), pageLinkOta); + Assert.assertEquals(0, pageDataOta.getTotalElements()); + } + + private void assertResourceIsDeleted(Tenant tenant, TbResource resource) { + assertThat(resourceService.findResourceById(tenant.getId(), resource.getId())) + .as("resource").isNull(); + PageLink pageLinkResources = new PageLink(1); + PageData tenantResources = + resourceService.findAllTenantResourcesByTenantId(tenant.getId(), pageLinkResources); + Assert.assertEquals(0, tenantResources.getTotalElements()); + } + + private void assertUserIsDeleted(Tenant tenant, User user) { + assertThat(userService.findUserById(tenant.getId(), user.getId())) + .as("user").isNull(); + PageLink pageLinkUsers = new PageLink(1); + PageData users = + userService.findUsersByTenantId(tenant.getId(), pageLinkUsers); + Assert.assertEquals(0, users.getTotalElements()); + } + + private void assertTenantAdminIsDeleted(Tenant savedTenant) { + PageLink pageLinkTenantAdmins = new PageLink(1); + PageData tenantAdmins = + userService.findTenantAdmins(savedTenant.getId(), pageLinkTenantAdmins); + Assert.assertEquals(0, tenantAdmins.getTotalElements()); + } + + private void assertEdgeIsDeleted(Tenant tenant, Edge edge) { + assertThat(edgeService.findEdgeById(tenant.getId(), edge.getId())) + .as("edge").isNull(); + PageLink pageLinkEdges = new PageLink(1); + PageData edges = edgeService.findEdgesByTenantId(tenant.getId(), pageLinkEdges); + Assert.assertEquals(0, edges.getTotalElements()); + } + + private void assertDashboardIsDeleted(Tenant tenant, Dashboard dashboard) { + assertThat(dashboardService.findDashboardById(tenant.getId(), dashboard.getId())) + .as("dashboard").isNull(); + PageLink pageLinkDashboards = new PageLink(1); + PageData dashboards = + dashboardService.findDashboardsByTenantId(tenant.getId(), pageLinkDashboards); + Assert.assertEquals(0, dashboards.getTotalElements()); + } + + private void assertDeviceProfileIsDeleted(Tenant tenant, DeviceProfile deviceProfile) { + assertThat(deviceProfileService.findDeviceProfileById(tenant.getId(), deviceProfile.getId())) + .as("deviceProfile").isNull(); + PageLink pageLinkDeviceProfiles = new PageLink(1); + PageData profiles = + deviceProfileService.findDeviceProfiles(tenant.getId(), pageLinkDeviceProfiles); + Assert.assertEquals(0, profiles.getTotalElements()); + } + + private void assertDeviceIsDeleted(Tenant tenant, Device device) { + assertThat(deviceService.findDeviceById(tenant.getId(), device.getId())) + .as("device").isNull(); + PageLink pageLinkDevices = new PageLink(1); + PageData devices = + deviceService.findDevicesByTenantId(tenant.getId(), pageLinkDevices); + Assert.assertEquals(0, devices.getTotalElements()); + } + + private void assertAssetIsDeleted(Tenant tenant, Asset asset) { + assertThat(assetService.findAssetById(tenant.getId(), asset.getId())) + .as("asset").isNull(); + PageLink pageLinkAssets = new PageLink(1); + PageData assets = + assetService.findAssetsByTenantId(tenant.getId(), pageLinkAssets); + Assert.assertEquals(0, assets.getTotalElements()); + } + + private void assertEntityViewIsDeleted(Tenant tenant, Device device, EntityView entityView) { + assertThat(entityViewService.findEntityViewById(tenant.getId(), entityView.getId())) + .as("entityView").isNull(); + List entityViews = + entityViewService.findEntityViewsByTenantIdAndEntityId(tenant.getId(), device.getId()); + Assert.assertTrue(entityViews.isEmpty()); + } + + private void assertWidgetsBundleIsDeleted(Tenant tenant, WidgetsBundle widgetsBundle) { + assertThat(widgetsBundleService.findWidgetsBundleById(tenant.getId(), widgetsBundle.getId())) + .as("widgetBundle").isNull(); + List widgetsBundlesByTenantId = + widgetsBundleService.findAllTenantWidgetsBundlesByTenantId(tenant.getId()); + Assert.assertTrue(widgetsBundlesByTenantId.isEmpty()); + } + + private void assertCustomerIsDeleted(Tenant tenant, Customer customer) { + assertThat(customerService.findCustomerById(tenant.getId(), customer.getId())) + .as("customer").isNull(); + PageLink pageLinkCustomer = new PageLink(1); + PageData pageDataCustomer = customerService + .findCustomersByTenantId(tenant.getId(), pageLinkCustomer); + Assert.assertEquals(0, pageDataCustomer.getTotalElements()); + } + + private Rpc createAndSaveRpcFor(Tenant tenant, Device device) { + Rpc rpc = new Rpc(); + rpc.setTenantId(tenant.getId()); + rpc.setDeviceId(device.getId()); + rpc.setStatus(RpcStatus.QUEUED); + rpc.setRequest(JacksonUtil.toJsonNode("{}")); + return rpcService.save(rpc); + } + + private TbResource createAndSaveResourceFor(Tenant tenant) { + TbResource resource = new TbResource(); + resource.setTenantId(tenant.getId()); + resource.setTitle("Test resource"); + resource.setResourceType(ResourceType.LWM2M_MODEL); + resource.setFileName("filename.txt"); + resource.setResourceKey("Test resource key"); + resource.setData("Some super test data"); + return resourceService.saveResource(resource); + } + + private OtaPackage createAndSaveOtaPackageFor(Tenant tenant, DeviceProfile deviceProfile) { + return otaPackageService.saveOtaPackage( + BaseOtaPackageServiceTest.createFirmware( + tenant.getId(), "2", deviceProfile.getId()) + ); + } + + private Edge createAndSaveEdgeFor(Tenant tenant) { + Edge edge = constructEdge(tenant.getId(), "Test edge", "Simple"); + return edgeService.saveEdge(edge); + } + + private RuleChain createAndSaveRuleChainFor(Tenant tenant) { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenant.getId()); + ruleChain.setName("Test rule chain"); + ruleChain.setType(RuleChainType.CORE); + return ruleChainService.saveRuleChain(ruleChain); + } + + private Dashboard createAndSaveDashboardFor(Tenant tenant, Customer customer) { + Dashboard dashboard = new Dashboard(); + dashboard.setTenantId(tenant.getId()); + dashboard.setTitle("Test dashboard"); + dashboard.setAssignedCustomers(Set.of(customer.toShortCustomerInfo())); + return dashboardService.saveDashboard(dashboard); + } + + private Asset createAndSaveAssetFor(Tenant tenant, Customer customer) { + Asset asset = new Asset(); + asset.setTenantId(tenant.getId()); + asset.setCustomerId(customer.getId()); + asset.setType("Test asset type"); + asset.setName("Test asset type"); + asset.setLabel("Test asset type"); + return assetService.saveAsset(asset); + } + + private EntityView createAndSaveEntityViewFor(Tenant tenant, Customer customer, Device device) { + EntityView entityView = new EntityView(); + entityView.setEntityId(device.getId()); + entityView.setTenantId(tenant.getId()); + entityView.setCustomerId(customer.getId()); + entityView.setType("Test type"); + entityView.setName("Test entity view"); + entityView.setStartTimeMs(0); + entityView.setEndTimeMs(840000); + return entityViewService.saveEntityView(entityView); + } + + private Device createAndSaveDeviceFor(Tenant tenant, Customer customer, DeviceProfile deviceProfile) { + Device device = new Device(); + device.setCustomerId(customer.getId()); + device.setTenantId(tenant.getId()); + device.setType("Test type"); + device.setName("TestType"); + device.setLabel("Test type"); + device.setDeviceProfileId(deviceProfile.getId()); + return deviceService.saveDevice(device); + } + + private DeviceProfile createAndSaveDeviceProfileWithProfileDataFor(Tenant tenant) { + DeviceProfile deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenant.getId()); + deviceProfile.setTransportType(DeviceTransportType.MQTT); + deviceProfile.setName("Test device profile"); + deviceProfile.setType(DeviceProfileType.DEFAULT); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setTransportConfiguration(new MqttDeviceProfileTransportConfiguration()); + deviceProfile.setProfileData(profileData); + return deviceProfileService.saveDeviceProfile(deviceProfile); + } + + private WidgetsBundle createAndSaveWidgetBundleFor(Tenant tenant) { + WidgetsBundle widgetsBundle = new WidgetsBundle(); + widgetsBundle.setTenantId(tenant.getId()); + widgetsBundle.setTitle("Test widgets bundle"); + widgetsBundle.setAlias("TestWidgetsBundle"); + widgetsBundle.setDescription("Just a simple widgets bundle"); + return widgetsBundleService.saveWidgetsBundle(widgetsBundle); + } + + private Customer createAndSaveCustomerFor(Tenant tenant) { + Customer customer = new Customer(); + customer.setTitle("Test customer"); + customer.setTenantId(tenant.getId()); + customer.setEmail("testCustomer@test.com"); + return customerService.saveCustomer(customer); + } + + private User createAndSaveUserFor(Tenant tenant) { + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setEmail("tenantAdmin@test.com"); + user.setFirstName("tenantAdmin"); + user.setLastName("tenantAdmin"); + user.setTenantId(tenant.getId()); + return userService.saveUser(user); + } + + private Tenant createAndSaveTenant(TenantProfile tenantProfile) { + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(tenantProfile.getId()); + return tenantService.saveTenant(tenant); + } + + private TenantProfile createAndSaveTenantProfile() { + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test tenant profile"); + return tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java index 4142034729..b38afe77f6 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java @@ -184,6 +184,140 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest { Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); } + @Test + public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS - 1L); + saveEntries(deviceId, TS); + saveEntries(deviceId, TS + 1L); + + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodEqualsInterval() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100L; i += 10L) { + saveEntries(deviceId, i); + } + saveEntries(deviceId, TS + 100L + 1L); + + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(1, entries.size()); + Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 100000L; i += 10000L) { + saveEntries(deviceId, i); + } + saveEntries(deviceId, TS + 100000L + 1L); + + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + saveEntries(deviceId, TS - 1L); + for (long i = TS; i <= TS + 80000L; i += 10000L) { + saveEntries(deviceId, i); + } + saveEntries(deviceId, TS + 80000L + 1L); + + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) { + saveEntries(deviceId, i); + } + + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1)); + } + + @Test + public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception { + DeviceId deviceId = new DeviceId(Uuids.timeBased()); + for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) { + saveEntries(deviceId, i); + } + + List queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER)); + + List entries = tsService.findAll(tenantId, deviceId, queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + + EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY)); + + entries = tsService.findAll(tenantId, entityView.getId(), queries).get(); + Assert.assertEquals(2, entries.size()); + Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0)); + Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1)); + } + @Test public void testFindByQueryDescOrder() throws Exception { DeviceId deviceId = new DeviceId(Uuids.timeBased()); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java new file mode 100644 index 0000000000..db216f7f6e --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java @@ -0,0 +1,156 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sqlts; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.ReadTsKvQuery; +import org.thingsboard.server.common.data.kv.TsKvEntry; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.BDDMockito.willCallRealMethod; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID; +import static org.thingsboard.server.common.data.kv.Aggregation.COUNT; + +public class AbstractChunkedAggregationTimeseriesDaoTest { + + final int LIMIT = 1; + final String TEMP = "temp"; + final String DESC = "DESC"; + AbstractChunkedAggregationTimeseriesDao tsDao; + + @Before + public void setUp() throws Exception { + tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class); + ListenableFuture> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class))); + willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any()); + willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any()); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull(); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT); + + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC); + ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT); + } + + @Test + public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() { + ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC); + willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query); + verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any()); + for (long i = 1; i <= 3000; i += 3) { + ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC); + verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT); + } + } + + long getTsForReadTsKvQuery(long startTs, long endTs) { + return startTs + (endTs - startTs) / 2L; + } + +} diff --git a/dao/src/test/resources/application-test.properties b/dao/src/test/resources/application-test.properties index 3ebcf5a7ec..063b1275d1 100644 --- a/dao/src/test/resources/application-test.properties +++ b/dao/src/test/resources/application-test.properties @@ -35,6 +35,9 @@ cache.specs.entityViews.maxSize=100000 cache.specs.claimDevices.timeToLiveInMinutes=1440 cache.specs.claimDevices.maxSize=100000 +cache.specs.tenants.timeToLiveInMinutes=1440 +cache.specs.tenants.maxSize=100000 + cache.specs.securitySettings.timeToLiveInMinutes=1440 cache.specs.securitySettings.maxSize=100000 diff --git a/dao/src/test/resources/sql/system-data.sql b/dao/src/test/resources/sql/system-data.sql index 1d0d29aec5..bfa33c2284 100644 --- a/dao/src/test/resources/sql/system-data.sql +++ b/dao/src/test/resources/sql/system-data.sql @@ -26,13 +26,13 @@ VALUES ( '61441950-4612-11e7-a919-92ebcb67fe33', 1592576748000, '5a797660-4612-1 '$2a$10$5JTB8/hxWc9WAy62nCGSxeefl3KWmipA9nFpVdDa0/xfIseeBB4Bu' ); /** System settings **/ -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'general', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6a2266e4-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'general', '{ "baseUrl": "http://localhost:8080" }' ); -INSERT INTO admin_settings ( id, created_time, key, json_value ) -VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, 'mail', '{ +INSERT INTO admin_settings ( id, created_time, tenant_id, key, json_value ) +VALUES ( '6eaaefa6-4612-11e7-a919-92ebcb67fe33', 1592576748000, '13814000-1dd2-11b2-8080-808080808080', 'mail', '{ "mailFrom": "Thingsboard ", "smtpProtocol": "smtp", "smtpHost": "localhost", diff --git a/docker/.env b/docker/.env index 11e44fdde8..7bc1a7a6a4 100644 --- a/docker/.env +++ b/docker/.env @@ -1,5 +1,8 @@ TB_QUEUE_TYPE=kafka +# redis or redis-cluster +CACHE=redis + DOCKER_REPO=thingsboard JS_EXECUTOR_DOCKER_NAME=tb-js-executor @@ -10,6 +13,7 @@ HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport +TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor TB_VERSION=latest diff --git a/docker/.gitignore b/docker/.gitignore index e5313d17cf..9c4c778f28 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -5,5 +5,5 @@ tb-node/db/** tb-node/postgres/** tb-node/cassandra/** tb-transports/*/log -docker/tb-vc-executor/log/** +tb-vc-executor/log/** !.env diff --git a/docker/README.md b/docker/README.md index 01f89ebb6c..71ea87f353 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env` **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details). +In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following: + +- `redis` - use Redis standalone cache (1 node - 1 master); +- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves); + +**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details). + Execute the following command to create log folders for the services and chown of these folders to the docker container users. To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env new file mode 100644 index 0000000000..a3b516063b --- /dev/null +++ b/docker/cache-redis-cluster.env @@ -0,0 +1,5 @@ +CACHE_TYPE=redis +REDIS_CONNECTION_TYPE=cluster +REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 +REDIS_USE_DEFAULT_POOL_CONFIG=false +REDIS_PASSWORD=thingsboard diff --git a/docker/cache-redis.env b/docker/cache-redis.env new file mode 100644 index 0000000000..7b92620666 --- /dev/null +++ b/docker/cache-redis.env @@ -0,0 +1,2 @@ +CACHE_TYPE=redis +REDIS_HOST=redis diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 550f2dfea4..28601afa3c 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -26,7 +26,7 @@ function additionalComposeArgs() { ADDITIONAL_COMPOSE_ARGS="-f docker-compose.hybrid.yml" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_ARGS @@ -55,7 +55,7 @@ function additionalComposeQueueArgs() { ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml" ;; *) - echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2 + echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent' or 'aws-sqs' or 'pubsub' or 'rabbitmq' or 'service-bus'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_QUEUE_ARGS @@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() { fi } +function additionalComposeCacheArgs() { + source .env + CACHE_COMPOSE_ARGS="" + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml" + ;; + redis-cluster) + CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" + ;; + *) + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 + exit 1 + esac + echo $CACHE_COMPOSE_ARGS +} + function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" case $DATABASE in postgres) - ADDITIONAL_STARTUP_SERVICES=postgres + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres" ;; hybrid) - ADDITIONAL_STARTUP_SERVICES="postgres cassandra" + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac + + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis" + ;; + redis-cluster) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" + ;; + *) + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 + exit 1 + esac + echo $ADDITIONAL_STARTUP_SERVICES } diff --git a/docker/docker-compose.aws-sqs.yml b/docker/docker-compose.aws-sqs.yml index e50d0d424f..331902851f 100644 --- a/docker/docker-compose.aws-sqs.yml +++ b/docker/docker-compose.aws-sqs.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-aws-sqs.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-aws-sqs.env + tb-vc-executor2: + env_file: + - queue-aws-sqs.env diff --git a/docker/docker-compose.cassandra.volumes.yml b/docker/docker-compose.cassandra.volumes.yml new file mode 100644 index 0000000000..37880b44fa --- /dev/null +++ b/docker/docker-compose.cassandra.volumes.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + cassandra: + volumes: + - cassandra-volume:/var/lib/cassandra + +volumes: + cassandra-volume: + external: true + name: ${CASSANDRA_DATA_VOLUME} diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index 6983dd3d3e..077acdca98 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -23,23 +23,15 @@ services: tb-core1: env_file: - queue-confluent.env - depends_on: - - redis tb-core2: env_file: - queue-confluent.env - depends_on: - - redis tb-rule-engine1: env_file: - queue-confluent.env - depends_on: - - redis tb-rule-engine2: env_file: - queue-confluent.env - depends_on: - - redis tb-mqtt-transport1: env_file: - queue-confluent.env @@ -61,3 +53,9 @@ services: tb-snmp-transport: env_file: - queue-confluent.env + tb-vc-executor1: + env_file: + - queue-confluent.env + tb-vc-executor2: + env_file: + - queue-confluent.env diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 7d7ab84c8f..12ea4cd099 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -29,7 +29,7 @@ services: - ./tb-node/postgres:/var/lib/postgresql/data cassandra: restart: always - image: "cassandra:3.11.3" + image: "cassandra:4.0.4" ports: - "9042" volumes: @@ -38,31 +38,23 @@ services: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - - redis - postgres - cassandra tb-core2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - - redis - postgres - cassandra tb-rule-engine1: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - - redis - postgres - cassandra tb-rule-engine2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - - redis - postgres - cassandra diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 2184528e53..e6fa0c489f 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -19,7 +19,7 @@ version: '2.2' services: kafka: restart: always - image: "wurstmeister/kafka:2.13-2.6.0" + image: "bitnami/kafka:3.2.0" ports: - "9092:9092" env_file: @@ -36,25 +36,21 @@ services: - queue-kafka.env depends_on: - kafka - - redis tb-core2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine1: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-mqtt-transport1: env_file: - queue-kafka.env @@ -90,3 +86,13 @@ services: - queue-kafka.env depends_on: - kafka + tb-vc-executor1: + env_file: + - queue-kafka.env + depends_on: + - kafka + tb-vc-executor2: + env_file: + - queue-kafka.env + depends_on: + - kafka diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 704e9400ae..caf78f23d7 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,59 +20,8 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb-core1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-core2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-coap-transport: - volumes: - - tb-coap-transport-log-volume:/var/log/tb-coap-transport - tb-lwm2m-transport: - volumes: - - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport - tb-http-transport1: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-http-transport2: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-mqtt-transport1: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-mqtt-transport2: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-snmp-transport: - volumes: - - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport volumes: postgres-db-volume: external: true name: ${POSTGRES_DATA_VOLUME} - tb-log-volume: - external: true - name: ${TB_LOG_VOLUME} - tb-coap-transport-log-volume: - external: true - name: ${TB_COAP_TRANSPORT_LOG_VOLUME} - tb-lwm2m-transport-log-volume: - external: true - name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} - tb-http-transport-log-volume: - external: true - name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} - tb-mqtt-transport-log-volume: - external: true - name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} - tb-snmp-transport-log-volume: - external: true - name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index 591ea59f7a..8fe8e6f53d 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -31,27 +31,19 @@ services: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-core2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine1: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index 0364957ee6..7c122d0835 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-pubsub.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-pubsub.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-pubsub.env + tb-vc-executor2: + env_file: + - queue-pubsub.env diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index 1eb37709e5..1f1cf1554c 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -23,59 +23,39 @@ services: tb-core1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-lwm2m-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-rabbitmq.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-rabbitmq.env + tb-vc-executor2: + env_file: + - queue-rabbitmq.env diff --git a/docker/docker-compose.redis-cluster.volumes.yml b/docker/docker-compose.redis-cluster.volumes.yml new file mode 100644 index 0000000000..2cf319bd21 --- /dev/null +++ b/docker/docker-compose.redis-cluster.volumes.yml @@ -0,0 +1,58 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + # Redis cluster + redis-node-0: + volumes: + - redis-cluster-data-0:/bitnami/redis/data + redis-node-1: + volumes: + - redis-cluster-data-1:/bitnami/redis/data + redis-node-2: + volumes: + - redis-cluster-data-2:/bitnami/redis/data + redis-node-3: + volumes: + - redis-cluster-data-3:/bitnami/redis/data + redis-node-4: + volumes: + - redis-cluster-data-4:/bitnami/redis/data + redis-node-5: + volumes: + - redis-cluster-data-5:/bitnami/redis/data + +volumes: + redis-cluster-data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster-data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster-data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster-data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster-data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster-data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml new file mode 100644 index 0000000000..55c1c3f1db --- /dev/null +++ b/docker/docker-compose.redis-cluster.yml @@ -0,0 +1,148 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis cluster + redis-node-0: + image: bitnami/redis-cluster:7.0 + volumes: + - ./tb-node/redis-cluster-data-0:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-1: + image: bitnami/redis-cluster:7.0 + volumes: + - ./tb-node/redis-cluster-data-1:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-2: + image: bitnami/redis-cluster:7.0 + volumes: + - ./tb-node/redis-cluster-data-2:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-3: + image: bitnami/redis-cluster:7.0 + volumes: + - ./tb-node/redis-cluster-data-3:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-4: + image: bitnami/redis-cluster:7.0 + volumes: + - ./tb-node/redis-cluster-data-4:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-5: + image: bitnami/redis-cluster:7.0 + volumes: + - ./tb-node/redis-cluster-data-5:/bitnami/redis/data + depends_on: + - redis-node-0 + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + environment: + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + - 'REDIS_CLUSTER_REPLICAS=1' + - 'REDIS_CLUSTER_CREATOR=yes' + +# ThingsBoard setup to use redis-cluster + tb-core1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-core2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-coap-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-lwm2m-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-snmp-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 diff --git a/docker/docker-compose.redis.volumes.yml b/docker/docker-compose.redis.volumes.yml new file mode 100644 index 0000000000..090aa441fe --- /dev/null +++ b/docker/docker-compose.redis.volumes.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + redis: + volumes: + - redis-data:/bitnami/redis/data + +volumes: + redis-data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml new file mode 100644 index 0000000000..e53a974134 --- /dev/null +++ b/docker/docker-compose.redis.yml @@ -0,0 +1,97 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis standalone + redis: + restart: always + image: bitnami/redis:7.0 + environment: + # ALLOW_EMPTY_PASSWORD is recommended only for development. + ALLOW_EMPTY_PASSWORD: "yes" + ports: + - '6379:6379' + volumes: + - ./tb-node/redis-data:/bitnami/redis/data + +# ThingsBoard setup to use redis-standalone + tb-core1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-core2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-coap-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-lwm2m-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-snmp-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor2: + env_file: + - cache-redis.env + depends_on: + - redis diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml index c511658b31..6e39de0baa 100644 --- a/docker/docker-compose.service-bus.yml +++ b/docker/docker-compose.service-bus.yml @@ -23,57 +23,39 @@ services: tb-core1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-core2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine1: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-rule-engine2: env_file: - queue-service-bus.env - depends_on: - - zookeeper - - redis tb-mqtt-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-mqtt-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport1: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-http-transport2: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-coap-transport: env_file: - queue-service-bus.env tb-lwm2m-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper tb-snmp-transport: env_file: - queue-service-bus.env - depends_on: - - zookeeper + tb-vc-executor1: + env_file: + - queue-service-bus.env + tb-vc-executor2: + env_file: + - queue-service-bus.env diff --git a/docker/docker-compose.volumes.yml b/docker/docker-compose.volumes.yml new file mode 100644 index 0000000000..58269473e4 --- /dev/null +++ b/docker/docker-compose.volumes.yml @@ -0,0 +1,81 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-core1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-coap-transport: + volumes: + - tb-coap-transport-log-volume:/var/log/tb-coap-transport + tb-lwm2m-transport: + volumes: + - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport + tb-http-transport1: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-http-transport2: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-mqtt-transport1: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-mqtt-transport2: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-snmp-transport: + volumes: + - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + +volumes: + tb-log-volume: + external: true + name: ${TB_LOG_VOLUME} + tb-coap-transport-log-volume: + external: true + name: ${TB_COAP_TRANSPORT_LOG_VOLUME} + tb-lwm2m-transport-log-volume: + external: true + name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} + tb-http-transport-log-volume: + external: true + name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} + tb-mqtt-transport-log-volume: + external: true + name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} + tb-snmp-transport-log-volume: + external: true + name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 82fbaeed74..3e0f21391d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,21 +20,17 @@ version: '2.2' services: zookeeper: restart: always - image: "zookeeper:3.5" + image: "zookeeper:3.8.0" ports: - "2181" environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - redis: - restart: always - image: redis:4.0 - ports: - - "6379" + ZOO_ADMINSERVER_ENABLED: "false" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" - scale: 20 + scale: 10 env_file: - tb-js-executor.env tb-core1: @@ -59,7 +55,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -85,7 +80,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -109,7 +103,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-rule-engine2: restart: always @@ -131,7 +124,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-mqtt-transport1: restart: always @@ -257,6 +249,38 @@ services: - "8080" env_file: - tb-web-ui.env + tb-vc-executor1: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor1 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 + tb-vc-executor2: + restart: always + image: "${DOCKER_REPO}/${TB_VC_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" + ports: + - "8081" + environment: + TB_SERVICE_ID: tb-vc-executor2 + env_file: + - tb-vc-executor.env + volumes: + - ./tb-vc-executor/conf:/config + - ./tb-vc-executor/log:/var/log/tb-vc-executor + depends_on: + - zookeeper + - tb-core1 + - tb-core2 haproxy: restart: always container_name: "${LOAD_BALANCER_NAME}" diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 5af0f96377..6224b7040b 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -15,7 +15,7 @@ # limitations under the License. # -mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/ +mkdir -p tb-node/log && sudo chown -R 799:799 tb-node/log mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log @@ -26,3 +26,28 @@ mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log + +mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log + +mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres + +mkdir -p tb-node/cassandra && sudo chown -R 999:999 tb-node/cassandra + +source .env +CACHE="${CACHE:-redis}" +case $CACHE in + redis) + mkdir -p tb-node/redis-data && sudo chown -R 1001:1001 tb-node/redis-data + ;; + redis-cluster) + mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-0 + mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-1 + mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-2 + mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-3 + mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-4 + mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-5 + ;; + *) + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 + exit 1 +esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 6f6b1e9511..9129c72796 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -45,12 +45,19 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ + tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 89f0f7844c..769150c1f3 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -23,6 +23,10 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 9f159774d8..9e5d08add9 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -23,6 +23,10 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 61e68d6dd5..5b09aea204 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -23,6 +23,10 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 739fcf6543..027280635d 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,11 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 3b1415a965..260e14cf8b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,10 +44,20 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull \ + tb-core1 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ + tb-core1 diff --git a/docker/kafka.env b/docker/kafka.env index bd23f99ad1..9c28885252 100644 --- a/docker/kafka.env +++ b/docker/kafka.env @@ -1,12 +1,11 @@ - -KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 -KAFKA_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 -KAFKA_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 -KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT -KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE -KAFKA_CREATE_TOPICS=js_eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb_transport.api.requests:3:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600 -KAFKA_AUTO_CREATE_TOPICS_ENABLE=false -KAFKA_LOG_RETENTION_BYTES=1073741824 -KAFKA_LOG_SEGMENT_BYTES=268435456 -KAFKA_LOG_RETENTION_MS=300000 -KAFKA_LOG_CLEANUP_POLICY=delete +ALLOW_PLAINTEXT_LISTENER=yes +KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 +KAFKA_CFG_LISTENERS=INSIDE://:9093,OUTSIDE://:9092 +KAFKA_CFG_ADVERTISED_LISTENERS=INSIDE://:9093,OUTSIDE://kafka:9092 +KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT +KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=false +KAFKA_CFG_INTER_BROKER_LISTENER_NAME=INSIDE +KAFKA_CFG_LOG_RETENTION_BYTES=1073741824 +KAFKA_CFG_SEGMENT_BYTES=268435456 +KAFKA_CFG_LOG_RETENTION_MS=300000 +KAFKA_CFG_LOG_CLEANUP_POLICY=delete diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index 9e6a41c930..079443c98b 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 1b4ce7a298..7e0679987f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000 METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-lwm2m-transport.env b/docker/tb-lwm2m-transport.env index 4616d45972..f284803a46 100644 --- a/docker/tb-lwm2m-transport.env +++ b/docker/tb-lwm2m-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 0cd51c7371..e38cb2124a 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-node.env b/docker/tb-node.env index e3393d41b1..ba66757ecc 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 JS_EVALUATOR=remote TRANSPORT_TYPE=remote -CACHE_TYPE=redis -REDIS_HOST=redis HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false diff --git a/docker/tb-snmp-transport.env b/docker/tb-snmp-transport.env index 4851e9f6c1..e2cc39d658 100644 --- a/docker/tb-snmp-transport.env +++ b/docker/tb-snmp-transport.env @@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-vc-executor.env b/docker/tb-vc-executor.env new file mode 100644 index 0000000000..f92e30b78f --- /dev/null +++ b/docker/tb-vc-executor.env @@ -0,0 +1,2 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 diff --git a/docker/tb-vc-executor/conf/logback.xml b/docker/tb-vc-executor/conf/logback.xml new file mode 100644 index 0000000000..dc95b3a885 --- /dev/null +++ b/docker/tb-vc-executor/conf/logback.xml @@ -0,0 +1,51 @@ + + + + + + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.log + + /var/log/tb-vc-executor/${TB_SERVICE_ID}/tb-vc-executor.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/docker/tb-vc-executor/conf/tb-vc-executor.conf b/docker/tb-vc-executor/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..f140e3fb76 --- /dev/null +++ b/docker/tb-vc-executor/conf/tb-vc-executor.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-vc-executor/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-vc-executor/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-vc-executor.out +export LOADER_PATH=/usr/share/tb-vc-executor/conf diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index 9c68789e67..a45badf44a 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -18,8 +18,17 @@ As result, in REPOSITORY column, next images should be present: thingsboard/tb-web-ui thingsboard/tb-js-executor -- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory: +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone: mvn clean install -DblackBoxTests.skip=false +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster: + + mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true + +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory in Hybrid mode (postgres + cassandra): + + mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.hybridMode=true + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index f60d6d7545..1fbf3755e6 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -27,6 +27,9 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.UUID; import static org.hamcrest.CoreMatchers.containsString; @@ -39,10 +42,12 @@ import static org.junit.Assert.fail; @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @Slf4j public class ContainerTestSuite { - + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); + final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; + private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; private static DockerComposeContainer testContainer; @@ -52,6 +57,8 @@ public class ContainerTestSuite { @ClassRule public static DockerComposeContainer getTestContainer() { if (testContainer == null) { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); + log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); try { final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; @@ -60,7 +67,7 @@ public class ContainerTestSuite { replaceInFile(targetDir + "docker-compose.yml", " container_name: \"${LOAD_BALANCER_NAME}\"", "", "container_name"); class DockerComposeContainerImpl> extends DockerComposeContainer { - public DockerComposeContainerImpl(File... composeFiles) { + public DockerComposeContainerImpl(List composeFiles) { super(composeFiles); } @@ -71,11 +78,26 @@ public class ContainerTestSuite { } } - testContainer = new DockerComposeContainerImpl<>( + List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), - new File(targetDir + "docker-compose.postgres.yml"), + new File(targetDir + "docker-compose.volumes.yml"), + IS_HYBRID_MODE + ? new File(targetDir + "docker-compose.hybrid.yml") + : new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose.kafka.yml")) + new File(targetDir + "docker-compose.kafka.yml"), + IS_REDIS_CLUSTER + ? new File(targetDir + "docker-compose.redis-cluster.yml") + : new File(targetDir + "docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml") + : new File(targetDir + "docker-compose.redis.volumes.yml"))); + + if (IS_HYBRID_MODE) { + composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + } + + testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) @@ -87,7 +109,9 @@ public class ContainerTestSuite { .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); + .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); } catch (Exception e) { log.error("Failed to create test container", e); fail("Failed to create test container"); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 417799f64f..b627d2787a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -15,63 +15,111 @@ */ package org.thingsboard.server.msa; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import java.io.File; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +@Slf4j public class ThingsBoardDbInstaller extends ExternalResource { + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); + final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + + private final static String CASSANDRA_DATA_VOLUME = "tb-cassandra-test-data-volume"; + private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; + private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume"; private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume"; private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; + private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; private final DockerComposeExecutor dockerCompose; private final String postgresDataVolume; + private final String cassandraDataVolume; + + private final String redisDataVolume; + private final String redisClusterDataVolume; private final String tbLogVolume; private final String tbCoapTransportLogVolume; private final String tbLwm2mTransportLogVolume; private final String tbHttpTransportLogVolume; private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; + private final String tbVcExecutorLogVolume; private final Map env; public ThingsBoardDbInstaller() { - List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), - new File("./../../docker/docker-compose.postgres.yml"), - new File("./../../docker/docker-compose.postgres.volumes.yml")); + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); + log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); + List composeFiles = new ArrayList<>(Arrays.asList( + new File("./../../docker/docker-compose.yml"), + new File("./../../docker/docker-compose.volumes.yml"), + IS_HYBRID_MODE + ? new File("./../../docker/docker-compose.hybrid.yml") + : new File("./../../docker/docker-compose.postgres.yml"), + new File("./../../docker/docker-compose.postgres.volumes.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") + )); + if (IS_HYBRID_MODE) { + composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); + } String identifier = Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase(); postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + cassandraDataVolume = project + "_" + CASSANDRA_DATA_VOLUME; + redisDataVolume = project + "_" + REDIS_DATA_VOLUME; + redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME; tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME; tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; + tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); env = new HashMap<>(); env.put("POSTGRES_DATA_VOLUME", postgresDataVolume); + if (IS_HYBRID_MODE) { + env.put("CASSANDRA_DATA_VOLUME", cassandraDataVolume); + } env.put("TB_LOG_VOLUME", tbLogVolume); env.put("TB_COAP_TRANSPORT_LOG_VOLUME", tbCoapTransportLogVolume); env.put("TB_LWM2M_TRANSPORT_LOG_VOLUME", tbLwm2mTransportLogVolume); env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); + env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); + } + } else { + env.put("REDIS_DATA_VOLUME", redisDataVolume); + } dockerCompose.withEnv(env); } @@ -86,6 +134,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + postgresDataVolume); dockerCompose.invokeDocker(); + if (IS_HYBRID_MODE) { + dockerCompose.withCommand("volume create " + cassandraDataVolume); + dockerCompose.invokeDocker(); + } + dockerCompose.withCommand("volume create " + tbLogVolume); dockerCompose.invokeDocker(); @@ -104,7 +157,26 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbSnmpTransportLogVolume); dockerCompose.invokeDocker(); - dockerCompose.withCommand("up -d redis postgres"); + dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); + dockerCompose.invokeDocker(); + + String additionalServices = ""; + if (IS_HYBRID_MODE) { + additionalServices += " cassandra"; + } + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + additionalServices = additionalServices + " redis-node-" + i; + dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); + dockerCompose.invokeDocker(); + } + } else { + additionalServices += " redis"; + dockerCompose.withCommand("volume create " + redisDataVolume); + dockerCompose.invokeDocker(); + } + + dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); @@ -126,10 +198,14 @@ public class ThingsBoardDbInstaller extends ExternalResource { copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/"); copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); + copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + + (IS_REDIS_CLUSTER + ? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining()) + : redisDataVolume)); dockerCompose.invokeDocker(); } diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 8f2ae8db49..86f441a5b3 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -32,15 +32,17 @@ const maxActiveScripts = Number(config.get('script.max_active_scripts')); const slowQueryLogMs = Number(config.get('script.slow_query_log_ms')); const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true'; -const {performance} = require('perf_hooks'); +const { performance } = require('node:perf_hooks'); function JsInvokeMessageProcessor(producer) { this.producer = producer; this.executor = new JsExecutor(useSandbox); this.scriptMap = new Map(); this.scriptIds = []; + this.executedScriptIdsCounter = []; this.executedScriptsCounter = 0; this.lastStatTime = performance.now(); + this.compilationTime = 0; } JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function (message) { @@ -125,7 +127,8 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, r const msSinceLastStat = nowMs - this.lastStatTime; const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000; this.lastStatTime = nowMs; - logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s]', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec); + logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s], compilation [%s]ms', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec, this.compilationTime); + this.compilationTime = 0; } if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { @@ -141,7 +144,7 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, r }, (err) => { var errorCode; - if (err.message.includes('Script execution timed out')) { + if (err && isString(err.message) && err.message.includes('Script execution timed out')) { errorCode = TIMEOUT_ERROR; } else { errorCode = RUNTIME_ERROR; @@ -167,6 +170,7 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function (requestId, var index = this.scriptIds.indexOf(scriptId); if (index > -1) { this.scriptIds.splice(index, 1); + this.executedScriptIdsCounter.splice(index, 1); } this.scriptMap.delete(scriptId); } @@ -198,14 +202,18 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scri return new Promise(function (resolve, reject) { const script = self.scriptMap.get(scriptId); if (script) { + incrementUseScriptId.call(self, scriptId); resolve(script); } else { + const startTime = performance.now(); self.executor.compileScript(scriptBody).then( (compiledScript) => { + self.compilationTime += (performance.now() - startTime); self.cacheScript(scriptId, compiledScript); resolve(compiledScript); }, (err) => { + self.compilationTime += (performance.now() - startTime); reject(err); } ); @@ -216,11 +224,10 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scri JsInvokeMessageProcessor.prototype.cacheScript = function (scriptId, script) { if (!this.scriptMap.has(scriptId)) { this.scriptIds.push(scriptId); + this.executedScriptIdsCounter.push(0); while (this.scriptIds.length > maxActiveScripts) { logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts); - const prevScriptId = this.scriptIds.shift(); - logger.info('Removing active script with id [%s]', prevScriptId); - this.scriptMap.delete(prevScriptId); + deleteMinUsedScript.apply(this); } } this.scriptMap.set(scriptId, script); @@ -291,4 +298,31 @@ function getScriptId(request) { return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB); } +function incrementUseScriptId(scriptId) { + const index = this.scriptIds.indexOf(scriptId); + if (this.executedScriptIdsCounter[index] < Number.MAX_SAFE_INTEGER) { + this.executedScriptIdsCounter[index]++; + } +} + +function deleteMinUsedScript() { + let min = Infinity; + let minIndex = 0; + const scriptIdsLength = this.executedScriptIdsCounter.length - 1; // ignored last added script + for (let i = 0; i < scriptIdsLength; i++) { + if (this.executedScriptIdsCounter[i] < min) { + min = this.executedScriptIdsCounter[i]; + minIndex = i; + } + } + const prevScriptId = this.scriptIds.splice(minIndex, 1)[0]; + this.executedScriptIdsCounter.splice(minIndex, 1) + logger.info('Removing active script with id [%s]', prevScriptId); + this.scriptMap.delete(prevScriptId); +} + +function isString(value) { + return typeof value === 'string'; +} + module.exports = JsInvokeMessageProcessor; diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 656779869d..1d1ad75318 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -const {logLevel, Kafka, CompressionTypes} = require('kafkajs'); +const {logLevel, Kafka, CompressionTypes, Partitioners} = require('kafkajs'); const config = require('config'), JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), @@ -159,7 +159,7 @@ async function sendMessagesAsBatch(isImmediately) { } consumer = kafkaClient.consumer({groupId: 'js-executor-group'}); - producer = kafkaClient.producer(); + producer = kafkaClient.producer({ createPartitioner: Partitioners.DefaultPartitioner }); /* //producer event instrumentation to debug diff --git a/msa/pom.xml b/msa/pom.xml index 106f2be2a3..dd40348bc8 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -40,6 +40,8 @@ tb + vc-executor + vc-executor-docker js-executor web-ui tb-node diff --git a/msa/tb/docker-cassandra/Dockerfile b/msa/tb/docker-cassandra/Dockerfile index 9d16d46ef5..f4d00a8d37 100644 --- a/msa/tb/docker-cassandra/Dockerfile +++ b/msa/tb/docker-cassandra/Dockerfile @@ -44,15 +44,17 @@ ENV CASSANDRA_LOG=/var/log/cassandra COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ RUN apt-get update \ - && apt-get install -y curl nmap procps gnupg2 \ + && apt-get install -y --no-install-recommends curl nmap procps gnupg2 \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null \ && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && echo 'deb http://downloads.apache.org/cassandra/debian 40x main' | tee --append /etc/apt/sources.list.d/cassandra.list > /dev/null \ && curl -L https://downloads.apache.org/cassandra/KEYS | apt-key add - \ && apt-get update \ - && apt-get install -y cassandra cassandra-tools postgresql-12 \ + && apt-get install -y --no-install-recommends cassandra cassandra-tools postgresql-${PG_MAJOR} \ + && rm -rf /var/lib/apt/lists/* \ && update-rc.d cassandra disable \ && update-rc.d postgresql disable \ + && apt-get purge -y --auto-remove \ && sed -i.old '/ulimit/d' /etc/init.d/cassandra \ && chmod a+x /tmp/*.sh \ && mv /tmp/start-tb.sh /usr/bin \ diff --git a/msa/tb/docker-postgres/Dockerfile b/msa/tb/docker-postgres/Dockerfile index 5dfdba41bd..90616c2ff0 100644 --- a/msa/tb/docker-postgres/Dockerfile +++ b/msa/tb/docker-postgres/Dockerfile @@ -36,12 +36,14 @@ ENV PGLOG=/var/log/postgres COPY logback.xml ${pkg.name}.conf start-db.sh stop-db.sh start-tb.sh upgrade-tb.sh install-tb.sh ${pkg.name}.deb /tmp/ RUN apt-get update \ - && apt-get install -y curl gnupg2 \ + && apt-get install -y --no-install-recommends curl gnupg2 \ && echo "deb http://apt.postgresql.org/pub/repos/apt/ $(. /etc/os-release && echo -n $VERSION_CODENAME)-pgdg main" | tee --append /etc/apt/sources.list.d/pgdg.list > /dev/null \ && curl -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ && apt-get update \ - && apt-get install -y postgresql-12 \ + && apt-get install -y --no-install-recommends postgresql-${PG_MAJOR} \ + && rm -rf /var/lib/apt/lists/* \ && update-rc.d postgresql disable \ + && apt-get purge -y --auto-remove \ && chmod a+x /tmp/*.sh \ && mv /tmp/start-tb.sh /usr/bin \ && mv /tmp/upgrade-tb.sh /usr/bin \ diff --git a/msa/vc-executor-docker/docker/Dockerfile b/msa/vc-executor-docker/docker/Dockerfile new file mode 100644 index 0000000000..af259d8f67 --- /dev/null +++ b/msa/vc-executor-docker/docker/Dockerfile @@ -0,0 +1,32 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk11:bullseye-slim + +COPY start-tb-vc-executor.sh ${pkg.name}.deb /tmp/ + +RUN mkdir -p /home/thingsboard/.config/jgit \ + && chown -R ${pkg.user}:${pkg.user} /home/thingsboard \ + && chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-vc-executor.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-vc-executor.sh"] diff --git a/msa/vc-executor-docker/docker/start-tb-vc-executor.sh b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh new file mode 100755 index 0000000000..4384008fd3 --- /dev/null +++ b/msa/vc-executor-docker/docker/start-tb-vc-executor.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER="/config" +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +export LOADER_PATH=/config,${LOADER_PATH} + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=/config/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml new file mode 100644 index 0000000000..39fc35f32a --- /dev/null +++ b/msa/vc-executor-docker/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor-docker + pom + + ThingsBoard Version Control Executor Microservice + https://thingsboard.io + ThingsBoard Version Control Executor Microservice + + + UTF-8 + ${basedir}/../.. + tb-vc-executor + tb-vc-executor + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard.msa + vc-executor + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-tb-vc-executor-deb + package + + copy + + + + + org.thingsboard.msa + vc-executor + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml new file mode 100644 index 0000000000..0125354371 --- /dev/null +++ b/msa/vc-executor/pom.xml @@ -0,0 +1,140 @@ + + + 4.0.0 + + + org.thingsboard + 3.4.0-SNAPSHOT + msa + + org.thingsboard.msa + vc-executor + + ThingsBoard Version Control Executor + https://thingsboard.io + Project for ThingsBoard version control microservice + + + UTF-8 + ${basedir}/../.. + java + false + process-resources + package + tb-vc-executor + false + ${project.build.directory}/windows + ThingsBoard Version Control Executor Service + org.thingsboard.server.vc.ThingsboardVersionControlExecutorApplication + + + + + org.thingsboard.common + queue + + + org.thingsboard.common + version-control + + + org.springframework.boot + spring-boot-starter-web + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-protobuf + + + io.grpc + grpc-stub + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + + + diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml new file mode 100644 index 0000000000..c0f33852a6 --- /dev/null +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -0,0 +1,45 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/msa/vc-executor/src/main/conf/tb-vc-executor.conf b/msa/vc-executor/src/main/conf/tb-vc-executor.conf new file mode 100644 index 0000000000..83287286bb --- /dev/null +++ b/msa/vc-executor/src/main/conf/tb-vc-executor.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java new file mode 100644 index 0000000000..6978f94b51 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/ThingsboardVersionControlExecutorApplication.java @@ -0,0 +1,47 @@ +package org.thingsboard.server.vc; /** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootApplication +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server", "org.thingsboard.server.common", "org.thingsboard.server.service.sync.vc"}) +public class ThingsboardVersionControlExecutorApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "tb-vc-executor"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardVersionControlExecutorApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java new file mode 100644 index 0000000000..0f4ac69948 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlQueueRoutingInfoService.java @@ -0,0 +1,31 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.queue.discovery.QueueRoutingInfo; +import org.thingsboard.server.queue.discovery.QueueRoutingInfoService; + +import java.util.Collections; +import java.util.List; + +@Service +public class VersionControlQueueRoutingInfoService implements QueueRoutingInfoService { + @Override + public List getAllQueuesRoutingInfo() { + return Collections.emptyList(); + } +} diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java new file mode 100644 index 0000000000..ebd7ed07b3 --- /dev/null +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.vc.service; + +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.queue.discovery.TenantRoutingInfo; +import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; + +@Service +public class VersionControlTenantRoutingInfoService implements TenantRoutingInfoService { + @Override + public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { + //This dummy implementation is ok since Version Control service does not produce any rule engine messages. + return new TenantRoutingInfo(tenantId, false, false); + } +} diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml new file mode 100644 index 0000000000..e25a1995c1 --- /dev/null +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -0,0 +1,38 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + \ No newline at end of file diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml new file mode 100644 index 0000000000..a97c4cfd37 --- /dev/null +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -0,0 +1,190 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# If you enabled process metrics you should also enable 'web-environment'. +spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" +# If you enabled process metrics you should set 'web-application-type' to 'servlet' value. +spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" + +spring.main.allow-circular-references: "true" + +server: + # Server bind address (has no effect if web-environment is disabled). + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port (has no effect if web-environment is disabled). + port: "${HTTP_BIND_PORT:8086}" + +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:true}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +queue: + type: "${TB_QUEUE_TYPE:kafka}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" + max.in.flight.requests.per.connection: "${TB_KAFKA_MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION:5}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + # Key-value properties for Kafka consumer per specific topic, e.g. tb_ota_package is a topic name for ota, tb_rule_engine.sq is a topic name for default SequentialByOriginator queue. + # Check TB_QUEUE_CORE_OTA_TOPIC and TB_QUEUE_RE_SQ_TOPIC params + consumer-properties-per-topic: + tb_ota_package: + - key: max.poll.records + value: "${TB_QUEUE_KAFKA_OTA_MAX_POLL_RECORDS:10}" + tb_version_control: + - key: max.poll.interval.ms + value: "${TB_QUEUE_KAFKA_VC_MAX_POLL_INTERVAL_MS:600000}" + # tb_rule_engine.sq: + # - key: max.poll.records + # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + other: # In this section you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside + - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms + value: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) + - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" + ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + ota: + topic: "${TB_QUEUE_CORE_OTA_TOPIC:tb_ota_package}" + pack-interval-ms: "${TB_QUEUE_CORE_OTA_PACK_INTERVAL_MS:60000}" + pack-size: "${TB_QUEUE_CORE_OTA_PACK_SIZE:100}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + vc: + topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" + partitions: "${TB_QUEUE_VC_PARTITIONS:10}" + poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + +vc: + # Pool size for handling export tasks + thread_pool_size: "${TB_VC_POOL_SIZE:2}" + git: + # Pool size for handling the git IO operations + io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}" + repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +service: + type: "${TB_SERVICE_TYPE:tb-vc-executor}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9273e7bebf..cb2264773d 100755 --- a/pom.xml +++ b/pom.xml @@ -39,12 +39,12 @@ 1.3.2 2.3.2 2.3.2 - 2.5.12 - 2.5.10 - 5.3.18 - 5.5.10 - 5.6.2 - 2.5.10 + 2.7.0 + 2.7.0 + 5.3.20 + 5.5.12 + 5.7.1 + 2.7.0 3.7.1 0.7.0 1.7.32 @@ -70,7 +70,7 @@ 2.2.6 3.0.0 2.0.0-M5 - 2.6.2 + 2.9.0 2.3.30 1.6.2 4.2.0 @@ -98,8 +98,9 @@ 5.0.2 0.2.1 - 2.8.0 + org.apache.kafka.common.network.NetworkReceive class in the application module. It addresses the issue https://issues.apache.org/jira/browse/KAFKA-4090. + Here is the source to track https://github.com/apache/kafka/tree/trunk/clients/src/main/java/org/apache/kafka/common/network --> + 3.2.0 4.1.1 2.57 2.7.7 @@ -112,28 +113,30 @@ 1.4.3 1.9.4 3.2.2 - 1.8.3 + 1.9.0 1.0.3TB 3.4.0 8.17.0 6.0.20.Final 3.0.0 2.0.1.Final - 1.6.4 + 1.6.8 2.8.5 4.1.0 4.3.1.0 2.7.2 1.5.2 - 5.7.2 + 5.8.2 2.6.0 + 5.13.1 1.3.0 1.2.7 1.16.0 1.12 3.0.0 + 6.1.0.202203080745-r 1.0.0 @@ -865,6 +868,11 @@ util ${project.version} + + org.thingsboard.common + version-control + ${project.version} + org.thingsboard.common cache @@ -1871,6 +1879,18 @@ ${zeroturnaround.version} test + + org.mock-server + mockserver-netty + ${mock-server.version} + test + + + org.mock-server + mockserver-client-java + ${mock-server.version} + test + org.opensmpp opensmpp-core @@ -1881,6 +1901,16 @@ aerogear-otp-java ${aerogear-otp.version} + + org.eclipse.jgit + org.eclipse.jgit + ${jgit.version} + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + ${jgit.version} + diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 2606fc32d4..ebbfffc1b3 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -136,6 +136,15 @@ awaitility test + + org.mock-server + mockserver-netty + + + org.mock-server + mockserver-client-java + + org.cassandraunit cassandra-unit diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java index b27caf6838..7b4f1792c4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCopyAttributesToEntityViewNode.java @@ -21,6 +21,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.CollectionsUtil; import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; import org.thingsboard.rule.engine.api.RuleNode; @@ -32,6 +33,7 @@ import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.objects.AttributesEntityView; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.session.SessionMsgType; @@ -140,15 +142,20 @@ public class TbCopyAttributesToEntityViewNode implements TbNode { } private boolean attributeContainsInEntityView(String scope, String attrKey, EntityView entityView) { + AttributesEntityView attributesEntityView = entityView.getKeys().getAttributes(); + List keys = null; switch (scope) { case DataConstants.CLIENT_SCOPE: - return entityView.getKeys().getAttributes().getCs().contains(attrKey); + keys = attributesEntityView.getCs(); + break; case DataConstants.SERVER_SCOPE: - return entityView.getKeys().getAttributes().getSs().contains(attrKey); + keys = attributesEntityView.getSs(); + break; case DataConstants.SHARED_SCOPE: - return entityView.getKeys().getAttributes().getSh().contains(attrKey); + keys = attributesEntityView.getSh(); + break; } - return false; + return CollectionsUtil.contains(keys, attrKey); } @Override diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java index 48f30314c5..d5213da4f2 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbCreateRelationNode.java @@ -124,7 +124,7 @@ public class TbCreateRelationNode extends TbAbstractRelationActionNode checkRelation(TbContext ctx, SearchDirectionIds sdId, String relationType) { - return ctx.getRelationService().checkRelation(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON); + return ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON); } private ListenableFuture processCreateRelation(TbContext ctx, EntityContainer entityContainer, SearchDirectionIds sdId, String relationType) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java index 361462bc3f..f19a271577 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbDeleteRelationNode.java @@ -98,7 +98,7 @@ public class TbDeleteRelationNode extends TbAbstractRelationActionNode processSingle(TbContext ctx, TbMsg msg, EntityContainer entityContainer, String relationType) { SearchDirectionIds sdId = processSingleSearchDirection(msg, entityContainer); - return Futures.transformAsync(ctx.getRelationService().checkRelation(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON), + return Futures.transformAsync(ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), sdId.getFromId(), sdId.getToId(), relationType, RelationTypeGroup.COMMON), result -> { if (result) { return processSingleDeleteRelation(ctx, sdId, relationType); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java index 1a26cf97ef..80b319e087 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java @@ -105,7 +105,7 @@ public class TbMsgGeneratorNode implements TbNode { public void onMsg(TbContext ctx, TbMsg msg) { log.trace("onMsg, config {}, msg {}", config, msg); if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) { - TbStopWatch sw = TbStopWatch.startNew(); + TbStopWatch sw = TbStopWatch.create(); withCallback(generate(ctx, msg), m -> { log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java index 398c4b77ec..c4bc9641a4 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbCheckRelationNode.java @@ -82,7 +82,7 @@ public class TbCheckRelationNode implements TbNode { to = EntityIdFactory.getByTypeAndId(config.getEntityType(), config.getEntityId()); from = msg.getOriginator(); } - return ctx.getRelationService().checkRelation(ctx.getTenantId(), from, to, config.getRelationType(), RelationTypeGroup.COMMON); + return ctx.getRelationService().checkRelationAsync(ctx.getTenantId(), from, to, config.getRelationType(), RelationTypeGroup.COMMON); } private ListenableFuture processList(TbContext ctx, TbMsg msg) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index f5752154c9..47d6af8e9c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonParseException; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.rule.engine.api.TbContext; @@ -50,7 +49,6 @@ import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; -@Slf4j public abstract class TbAbstractGetAttributesNode implements TbNode { private static ObjectMapper mapper = new ObjectMapper(); @@ -113,7 +111,6 @@ public abstract class TbAbstractGetAttributesNode> attributeKvEntryListFuture = ctx.getAttributesService().find(ctx.getTenantId(), entityId, scope, keys); return Futures.transform(attributeKvEntryListFuture, attributeKvEntryList -> { - log.warn("[{}][{}][{}][{}] Lookup attribute result: {}", ctx.getTenantId(), entityId, scope, keys, attributeKvEntryList); if (!CollectionUtils.isEmpty(attributeKvEntryList)) { List existingAttributesKvEntry = attributeKvEntryList.stream().filter(attributeKvEntry -> keys.contains(attributeKvEntry.getKey())).collect(Collectors.toList()); existingAttributesKvEntry.forEach(kvEntry -> msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString())); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java index 1d139c9b8d..f0bbdfb66b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/profile/DeviceState.java @@ -197,7 +197,8 @@ class DeviceState { private void processAlarmDeleteNotification(TbContext ctx, TbMsg msg) { Alarm alarm = JacksonUtil.fromString(msg.getData(), Alarm.class); - alarmStates.values().removeIf(alarmState -> alarmState.getCurrentAlarm().getId().equals(alarm.getId())); + alarmStates.values().removeIf(alarmState -> alarmState.getCurrentAlarm() != null + && alarmState.getCurrentAlarm().getId().equals(alarm.getId())); ctx.tellSuccess(msg); } @@ -219,11 +220,15 @@ class DeviceState { } if (!keys.isEmpty()) { EntityKeyType keyType = getKeyTypeFromScope(scope); - keys.forEach(key -> latestValues.removeValue(new EntityKey(keyType, key))); + Set removedKeys = keys.stream().map(key -> new EntityKey(keyType, key)) + .peek(latestValues::removeValue) + .map(DataSnapshot::toConditionKey).collect(Collectors.toSet()); + SnapshotUpdate update = new SnapshotUpdate(AlarmConditionKeyType.ATTRIBUTE, removedKeys); + for (DeviceProfileAlarm alarm : deviceProfile.getAlarmSettings()) { AlarmState alarmState = alarmStates.computeIfAbsent(alarm.getId(), a -> new AlarmState(this.deviceProfile, deviceId, alarm, getOrInitPersistedAlarmState(alarm), dynamicPredicateValueCtx)); - stateChanged |= alarmState.process(ctx, msg, latestValues, null); + stateChanged |= alarmState.process(ctx, msg, latestValues, update); } } ctx.tellSuccess(msg); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index c2a6867f94..e035d415ed 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -40,6 +40,7 @@ import org.springframework.util.concurrent.ListenableFuture; import org.springframework.util.concurrent.ListenableFutureCallback; import org.springframework.web.client.AsyncRestTemplate; import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TbRelationTypes; @@ -54,6 +55,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; import java.net.Authenticator; import java.net.PasswordAuthentication; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Deque; @@ -189,8 +191,9 @@ public class TbHttpClient { entity = new HttpEntity<>(msg.getData(), headers); } + URI uri = buildEncodedUri(endpointUrl); ListenableFuture> future = httpClient.exchange( - endpointUrl, method, entity, String.class); + uri, method, entity, String.class); future.addCallback(new ListenableFutureCallback>() { @Override public void onFailure(Throwable throwable) { @@ -214,6 +217,28 @@ public class TbHttpClient { } } + public URI buildEncodedUri(String endpointUrl) { + if (endpointUrl == null) { + throw new RuntimeException("Url string cannot be null!"); + } + if (endpointUrl.isEmpty()) { + throw new RuntimeException("Url string cannot be empty!"); + } + + URI uri = UriComponentsBuilder.fromUriString(endpointUrl).build().encode().toUri(); + if (uri.getScheme() == null || uri.getScheme().isEmpty()) { + throw new RuntimeException("Transport scheme(protocol) must be provided!"); + } + + boolean authorityNotValid = uri.getAuthority() == null || uri.getAuthority().isEmpty(); + boolean hostNotValid = uri.getHost() == null || uri.getHost().isEmpty(); + if (authorityNotValid || hostNotValid) { + throw new RuntimeException("Url string is invalid!"); + } + + return uri; + } + private TbMsg processResponse(TbContext ctx, TbMsg origMsg, ResponseEntity response) { TbMsgMetaData metaData = origMsg.getMetaData(); metaData.putValue(STATUS, response.getStatusCode().name()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java index b2a0d8dfc9..eafff21819 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbCreateRelationNodeTest.java @@ -113,7 +113,7 @@ public class TbCreateRelationNodeTest { metaData.putValue("type", "AssetType"); msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); @@ -144,7 +144,7 @@ public class TbCreateRelationNodeTest { when(ctx.getRelationService().findByToAndTypeAsync(any(), eq(msg.getOriginator()), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(Collections.singletonList(relation))); when(ctx.getRelationService().deleteRelationAsync(any(), eq(relation))).thenReturn(Futures.immediateFuture(true)); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); @@ -171,7 +171,7 @@ public class TbCreateRelationNodeTest { metaData.putValue("type", "AssetType"); msg = TbMsg.newMsg(DataConstants.ENTITY_CREATED, deviceId, metaData, TbMsgDataType.JSON, "{}", ruleChainId, ruleNodeId); - when(ctx.getRelationService().checkRelation(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) + when(ctx.getRelationService().checkRelationAsync(any(), eq(assetId), eq(deviceId), eq(RELATION_TYPE_CONTAINS), eq(RelationTypeGroup.COMMON))) .thenReturn(Futures.immediateFuture(false)); when(ctx.getRelationService().saveRelationAsync(any(), eq(new EntityRelation(assetId, deviceId, RELATION_TYPE_CONTAINS, RelationTypeGroup.COMMON)))) .thenReturn(Futures.immediateFuture(true)); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java new file mode 100644 index 0000000000..0f7795ab9d --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/DeviceStateTest.java @@ -0,0 +1,176 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.profile; + +import com.google.common.util.concurrent.Futures; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.RuleEngineAlarmService; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.device.profile.AlarmCondition; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilter; +import org.thingsboard.server.common.data.device.profile.AlarmConditionFilterKey; +import org.thingsboard.server.common.data.device.profile.AlarmConditionKeyType; +import org.thingsboard.server.common.data.device.profile.AlarmRule; +import org.thingsboard.server.common.data.device.profile.DeviceProfileAlarm; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.device.profile.SimpleAlarmConditionSpec; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.query.BooleanFilterPredicate; +import org.thingsboard.server.common.data.query.EntityKeyValueType; +import org.thingsboard.server.common.data.query.FilterPredicateValue; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.session.SessionMsgType; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.device.DeviceService; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DeviceStateTest { + + private TbContext ctx; + + @Before + public void beforeEach() { + ctx = mock(TbContext.class); + + when(ctx.getDeviceService()).thenReturn(mock(DeviceService.class)); + + AttributesService attributesService = mock(AttributesService.class); + when(attributesService.find(any(), any(), any(), anyCollection())).thenReturn(Futures.immediateFuture(Collections.emptyList())); + when(ctx.getAttributesService()).thenReturn(attributesService); + + RuleEngineAlarmService alarmService = mock(RuleEngineAlarmService.class); + when(alarmService.findLatestByOriginatorAndType(any(), any(), any())).thenReturn(Futures.immediateFuture(null)); + when(alarmService.createOrUpdateAlarm(any())).thenAnswer(invocationOnMock -> { + Alarm alarm = invocationOnMock.getArgument(0); + alarm.setId(new AlarmId(UUID.randomUUID())); + return alarm; + }); + when(ctx.getAlarmService()).thenReturn(alarmService); + + when(ctx.newMsg(any(), any(), any(), any(), any(), any())).thenAnswer(invocationOnMock -> { + String data = invocationOnMock.getArgument(invocationOnMock.getArguments().length - 1); + return TbMsg.newMsg(null, null, new TbMsgMetaData(), data); + }); + + } + + @Test + public void whenAttributeIsDeleted_thenUnneededAlarmRulesAreNotReevaluated() throws Exception { + + DeviceProfileAlarm alarmConfig = createAlarmConfigWithBoolAttrCondition("enabled", false); + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + DeviceState deviceState = createDeviceState(deviceId, alarmConfig); + + TbMsg attributeUpdateMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), + deviceId, new TbMsgMetaData(), "{ \"enabled\": false }"); + + deviceState.process(ctx, attributeUpdateMsg); + + ArgumentCaptor resultMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); + verify(ctx).enqueueForTellNext(resultMsgCaptor.capture(), eq("Alarm Created")); + Alarm alarm = JacksonUtil.fromString(resultMsgCaptor.getValue().getData(), Alarm.class); + + deviceState.process(ctx, TbMsg.newMsg(DataConstants.ALARM_CLEAR, deviceId, new TbMsgMetaData(), JacksonUtil.toString(alarm))); + reset(ctx); + + String deletedAttributes = "{ \"attributes\": [ \"other\" ] }"; + deviceState.process(ctx, TbMsg.newMsg(DataConstants.ATTRIBUTES_DELETED, deviceId, new TbMsgMetaData(), deletedAttributes)); + verify(ctx, never()).enqueueForTellNext(any(), anyString()); + } + + @Test + public void whenDeletingClearedAlarm_thenNoError() throws Exception { + DeviceProfileAlarm alarmConfig = createAlarmConfigWithBoolAttrCondition("enabled", false); + DeviceId deviceId = new DeviceId(UUID.randomUUID()); + DeviceState deviceState = createDeviceState(deviceId, alarmConfig); + + TbMsg attributeUpdateMsg = TbMsg.newMsg(SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), + deviceId, new TbMsgMetaData(), "{ \"enabled\": false }"); + + deviceState.process(ctx, attributeUpdateMsg); + ArgumentCaptor resultMsgCaptor = ArgumentCaptor.forClass(TbMsg.class); + verify(ctx).enqueueForTellNext(resultMsgCaptor.capture(), eq("Alarm Created")); + Alarm alarm = JacksonUtil.fromString(resultMsgCaptor.getValue().getData(), Alarm.class); + + deviceState.process(ctx, TbMsg.newMsg(DataConstants.ALARM_CLEAR, deviceId, new TbMsgMetaData(), JacksonUtil.toString(alarm))); + + TbMsg alarmDeleteNotification = TbMsg.newMsg(DataConstants.ALARM_DELETE, deviceId, new TbMsgMetaData(), JacksonUtil.toString(alarm)); + assertDoesNotThrow(() -> { + deviceState.process(ctx, alarmDeleteNotification); + }); + } + + + private DeviceState createDeviceState(DeviceId deviceId, DeviceProfileAlarm... alarmConfigs) { + DeviceProfile deviceProfile = new DeviceProfile(); + DeviceProfileData profileData = new DeviceProfileData(); + profileData.setAlarms(List.of(alarmConfigs)); + deviceProfile.setProfileData(profileData); + + ProfileState profileState = new ProfileState(deviceProfile); + return new DeviceState(ctx, new TbDeviceProfileNodeConfiguration(), + deviceId, profileState, null); + } + + private DeviceProfileAlarm createAlarmConfigWithBoolAttrCondition(String key, boolean value) { + + AlarmConditionFilter condition = new AlarmConditionFilter(); + condition.setKey(new AlarmConditionFilterKey(AlarmConditionKeyType.ATTRIBUTE, key)); + condition.setValueType(EntityKeyValueType.BOOLEAN); + BooleanFilterPredicate predicate = new BooleanFilterPredicate(); + predicate.setOperation(BooleanFilterPredicate.BooleanOperation.EQUAL); + predicate.setValue(new FilterPredicateValue<>(value)); + condition.setPredicate(predicate); + + DeviceProfileAlarm alarmConfig = new DeviceProfileAlarm(); + alarmConfig.setId("MyAlarmID"); + alarmConfig.setAlarmType("MyAlarm"); + AlarmRule alarmRule = new AlarmRule(); + AlarmCondition alarmCondition = new AlarmCondition(); + alarmCondition.setSpec(new SimpleAlarmConditionSpec()); + alarmCondition.setCondition(List.of(condition)); + alarmRule.setCondition(alarmCondition); + alarmConfig.setCreateRules(new TreeMap<>(Map.of(AlarmSeverity.CRITICAL, alarmRule))); + + return alarmConfig; + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java index d1ac360d44..315550a287 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/profile/TbDeviceProfileNodeTest.java @@ -49,7 +49,6 @@ import org.thingsboard.server.common.data.device.profile.RepeatingAlarmCondition import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.query.BooleanFilterPredicate; @@ -1109,7 +1108,7 @@ public class TbDeviceProfileNodeTest { AttributeKvEntity attributeKvEntityActiveSchedule = new AttributeKvEntity(); attributeKvEntityActiveSchedule.setId(compositeKeyActiveSchedule); attributeKvEntityActiveSchedule.setJsonValue( - "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":8.64e+7,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" + "{\"timezone\":\"Europe/Kiev\",\"items\":[{\"enabled\":true,\"dayOfWeek\":1,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":2,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":3,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":4,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":5,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":6,\"startsOn\":0,\"endsOn\":8.64e+7},{\"enabled\":true,\"dayOfWeek\":7,\"startsOn\":0,\"endsOn\":8.64e+7}],\"dynamicValue\":null}" ); attributeKvEntityActiveSchedule.setLastUpdateTs(0L); @@ -1166,6 +1165,8 @@ public class TbDeviceProfileNodeTest { TbMsg msg = TbMsg.newMsg(SessionMsgType.POST_TELEMETRY_REQUEST.name(), deviceId, new TbMsgMetaData(), TbMsgDataType.JSON, mapper.writeValueAsString(data), null, null); +// Mockito.reset(ctx); + node.onMsg(ctx, msg); verify(ctx).tellSuccess(msg); verify(ctx).enqueueForTellNext(theMsg, "Alarm Created"); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java index 9834ef631d..f7a1d83ac9 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java @@ -18,16 +18,38 @@ package org.thingsboard.rule.engine.rest; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import org.awaitility.Awaitility; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockserver.integration.ClientAndServer; +import org.springframework.web.client.AsyncRestTemplate; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import java.net.URI; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.BDDMockito.willCallRealMethod; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockserver.integration.ClientAndServer.startClientAndServer; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; public class TbHttpClientTest { @@ -58,4 +80,125 @@ public class TbHttpClientTest { eventLoop = client.getSharedOrCreateEventLoopGroup(null); assertThat(eventLoop, instanceOf(NioEventLoopGroup.class)); } + + @Test + public void testBuildSimpleUri() { + Mockito.when(client.buildEncodedUri(any())).thenCallRealMethod(); + String url = "http://localhost:8080/"; + URI uri = client.buildEncodedUri(url); + Assert.assertEquals(url, uri.toString()); + } + + @Test + public void testBuildUriWithoutProtocol() { + Mockito.when(client.buildEncodedUri(any())).thenCallRealMethod(); + String url = "localhost:8080/"; + assertThatThrownBy(() -> client.buildEncodedUri(url)); + } + + @Test + public void testBuildInvalidUri() { + Mockito.when(client.buildEncodedUri(any())).thenCallRealMethod(); + String url = "aaa"; + assertThatThrownBy(() -> client.buildEncodedUri(url)); + } + + @Test + public void testBuildUriWithSpecialSymbols() { + Mockito.when(client.buildEncodedUri(any())).thenCallRealMethod(); + String url = "http://192.168.1.1/data?d={\"a\": 12}"; + String expected = "http://192.168.1.1/data?d=%7B%22a%22:%2012%7D"; + URI uri = client.buildEncodedUri(url); + Assert.assertEquals(expected, uri.toString()); + } + + @Test + public void testProcessMessageWithJsonInUrlVariable() throws Exception { + String host = "localhost"; + String path = "/api"; + String paramKey = "data"; + String paramVal = "[{\"test\":\"test\"}]"; + String successResponseBody = "SUCCESS"; + + var server = setUpDummyServer(host, path, paramKey, paramVal, successResponseBody); + + String endpointUrl = String.format( + "http://%s:%d%s?%s=%s", + host, server.getPort(), path, paramKey, paramVal + ); + String method = "GET"; + + + var config = new TbRestApiCallNodeConfiguration() + .defaultConfiguration(); + config.setRequestMethod(method); + config.setRestEndpointUrlPattern(endpointUrl); + config.setUseSimpleClientHttpFactory(true); + + var asyncRestTemplate = new AsyncRestTemplate(); + + var httpClient = new TbHttpClient(config, eventLoop); + httpClient.setHttpClient(asyncRestTemplate); + + var msg = TbMsg.newMsg("GET", new DeviceId(EntityId.NULL_UUID), TbMsgMetaData.EMPTY, "{}"); + var successMsg = TbMsg.newMsg( + "SUCCESS", msg.getOriginator(), + msg.getMetaData(), msg.getData() + ); + + var ctx = mock(TbContext.class); + when(ctx.transformMsg( + eq(msg), eq(msg.getType()), + eq(msg.getOriginator()), + eq(msg.getMetaData()), + eq(msg.getData()) + )).thenReturn(successMsg); + + var capturedData = ArgumentCaptor.forClass(String.class); + + when(ctx.transformMsg( + eq(msg), eq(msg.getType()), + eq(msg.getOriginator()), + any(), + capturedData.capture() + )).thenReturn(successMsg); + + httpClient.processMessage(ctx, msg); + + Awaitility.await() + .atMost(30, TimeUnit.SECONDS) + .until(() -> { + try { + verify(ctx, times(1)).tellSuccess(any()); + return true; + } catch (Exception e) { + return false; + } + }); + + verify(ctx, times(1)).tellSuccess(any()); + verify(ctx, times(0)).tellFailure(any(), any()); + Assert.assertEquals(successResponseBody, capturedData.getValue()); + } + + private ClientAndServer setUpDummyServer(String host, String path, String paramKey, String paramVal, String successResponseBody) { + var server = startClientAndServer(host, 1080); + createGetMethodExpectations(server, path, paramKey, paramVal, successResponseBody); + return server; + } + + private void createGetMethodExpectations(ClientAndServer server, String path, String paramKey, String paramVal, String successResponseBody) { + server.when( + request() + .withMethod("GET") + .withPath(path) + .withQueryStringParameter(paramKey, paramVal) + ).respond( + response() + .withStatusCode(200) + .withBody(successResponseBody) + ); + } + + } \ No newline at end of file diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 5262fbaef1..2d408e0008 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -19,6 +19,8 @@ spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" +spring.main.allow-circular-references: "true" + server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -149,6 +151,7 @@ queue: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 7574687ecc..8e8cbc0641 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -14,6 +14,8 @@ # limitations under the License. # +spring.main.allow-circular-references: "true" + server: # Server bind address address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -136,6 +138,7 @@ queue: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" diff --git a/transport/lwm2m/src/main/conf/logback.xml b/transport/lwm2m/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/lwm2m/src/main/conf/logback.xml +++ b/transport/lwm2m/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/lwm2m/src/main/resources/logback.xml b/transport/lwm2m/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/lwm2m/src/main/resources/logback.xml +++ b/transport/lwm2m/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 19d70e0bc2..fb273a6118 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -19,6 +19,8 @@ spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" +spring.main.allow-circular-references: "true" + server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -214,6 +216,7 @@ queue: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 1a6f04e598..3514449861 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -19,6 +19,8 @@ spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" +spring.main.allow-circular-references: "true" + server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -166,6 +168,7 @@ queue: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/snmp/src/main/conf/logback.xml +++ b/transport/snmp/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/snmp/src/main/resources/logback.xml +++ b/transport/snmp/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 6ff1a37442..5f95ad382d 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -19,6 +19,8 @@ spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" +spring.main.allow-circular-references: "true" + server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -112,6 +114,7 @@ queue: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" retries: "${TB_KAFKA_RETRIES:1}" + compression.type: "${TB_KAFKA_COMPRESSION_TYPE:none}" # none or gzip batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" linger.ms: "${TB_KAFKA_LINGER_MS:1}" max.request.size: "${TB_KAFKA_MAX_REQUEST_SIZE:1048576}" diff --git a/ui-ngx/angular.json b/ui-ngx/angular.json index f4d4412723..da4d21a84b 100644 --- a/ui-ngx/angular.json +++ b/ui-ngx/angular.json @@ -79,7 +79,8 @@ "node_modules/leaflet.markercluster/dist/MarkerCluster.Default.css", "node_modules/@geoman-io/leaflet-geoman-free/dist/leaflet-geoman.css", "node_modules/prismjs/themes/prism.css", - "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css" + "node_modules/prismjs/plugins/line-numbers/prism-line-numbers.css", + "node_modules/ace-diff/dist/ace-diff.min.css" ], "stylePreprocessorOptions": { "includePaths": [ @@ -130,7 +131,8 @@ "jstree", "qrcode", "wcwidth", - "leaflet-polylinedecorator" + "leaflet-polylinedecorator", + "ace-diff" ] }, "configurations": { diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 53b7fc6190..c9db64ddc2 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -41,6 +41,7 @@ "@ngx-translate/core": "^13.0.0", "@ngx-translate/http-loader": "^6.0.0", "ace-builds": "^1.4.13", + "ace-diff": "^3.0.3", "angular-gridster2": "~12.1.1", "angular2-hotkeys": "^2.4.0", "canvas-gauges": "^2.1.7", @@ -106,6 +107,7 @@ "@angular/compiler-cli": "^12.2.13", "@angular/language-service": "^12.2.13", "@ngtools/webpack": "~12.2.13", + "@types/ace-diff": "^2.1.1", "@types/canvas-gauges": "^2.1.4", "@types/flot": "^0.0.32", "@types/jasmine": "~3.10.2", diff --git a/ui-ngx/src/app/core/api/widget-subscription.ts b/ui-ngx/src/app/core/api/widget-subscription.ts index 635e98d89c..41aeb5dd14 100644 --- a/ui-ngx/src/app/core/api/widget-subscription.ts +++ b/ui-ngx/src/app/core/api/widget-subscription.ts @@ -53,12 +53,14 @@ import { forkJoin, Observable, of, ReplaySubject, Subject, throwError, timer } f import { CancelAnimationFrame } from '@core/services/raf.service'; import { EntityType } from '@shared/models/entity-type.models'; import { - createLabelFromDatasource, createLabelFromPattern, - deepClone, flatFormattedData, + createLabelFromPattern, + deepClone, + flatFormattedData, formattedDataFormDatasourceData, isDefined, isDefinedAndNotNull, - isEqual + isEqual, + parseHttpErrorMessage } from '@core/utils'; import { EntityId } from '@app/shared/models/id/entity-id'; import * as moment_ from 'moment'; @@ -283,7 +285,8 @@ export class WidgetSubscription implements IWidgetSubscription { (this.legendConfig.showMin === true || this.legendConfig.showMax === true || this.legendConfig.showAvg === true || - this.legendConfig.showTotal === true); + this.legendConfig.showTotal === true || + this.legendConfig.showLatest === true); this.initDataSubscription().subscribe(() => { subscriptionSubject.next(this); subscriptionSubject.complete(); @@ -801,10 +804,10 @@ export class WidgetSubscription implements IWidgetSubscription { this.rpcErrorText = 'Request Timeout.'; } else { this.rpcErrorText = 'Error : ' + rejection.status + ' - ' + rejection.statusText; - const error = this.extractRejectionErrorText(rejection); + const error = parseHttpErrorMessage(rejection, this.ctx.translate); if (error) { this.rpcErrorText += '
'; - this.rpcErrorText += error; + this.rpcErrorText += error.message; } } this.callbacks.onRpcFailed(this); @@ -816,40 +819,6 @@ export class WidgetSubscription implements IWidgetSubscription { } } - private extractRejectionErrorText(rejection: HttpErrorResponse) { - let error = null; - if (rejection.error) { - error = rejection.error; - try { - error = rejection.error ? JSON.parse(rejection.error) : null; - } catch (e) {} - } - if (error && !error.message) { - error = this.prepareMessageFromData(error); - } else if (error && error.message) { - error = error.message; - } - return error; - } - - private prepareMessageFromData(data) { - if (typeof data === 'object' && data.constructor === ArrayBuffer) { - const msg = String.fromCharCode.apply(null, new Uint8Array(data)); - try { - const msgObj = JSON.parse(msg); - if (msgObj.message) { - return msgObj.message; - } else { - return msg; - } - } catch (e) { - return msg; - } - } else { - return data; - } - } - update(isTimewindowTypeChanged = false) { if (this.type !== widgetType.rpc) { this.widgetTimewindowChangedSubject.next(this.timeWindowConfig); @@ -1327,6 +1296,7 @@ export class WidgetSubscription implements IWidgetSubscription { max: null, avg: null, total: null, + latest: null, hidden: false }; this.legendData.data.push(legendKeyData); @@ -1558,6 +1528,9 @@ export class WidgetSubscription implements IWidgetSubscription { if (this.legendConfig.showTotal) { legendKeyData.total = this.ctx.widgetUtils.formatValue(calculateTotal(data), decimals, units); } + if (this.legendConfig.showLatest) { + legendKeyData.latest = this.ctx.widgetUtils.formatValue(calculateLatest(data), decimals, units); + } this.callbacks.legendDataUpdated(this, detectChanges !== false); } @@ -1630,3 +1603,11 @@ function calculateTotal(data: DataSet): number { return null; } } + +function calculateLatest(data: DataSet): number { + if (data.length > 0) { + return Number(data[data.length - 1][1]); + } else { + return null; + } +} diff --git a/ui-ngx/src/app/core/auth/auth.actions.ts b/ui-ngx/src/app/core/auth/auth.actions.ts index 6edbdcd5a2..a60720e61d 100644 --- a/ui-ngx/src/app/core/auth/auth.actions.ts +++ b/ui-ngx/src/app/core/auth/auth.actions.ts @@ -23,7 +23,8 @@ export enum AuthActionTypes { UNAUTHENTICATED = '[Auth] Unauthenticated', LOAD_USER = '[Auth] Load User', UPDATE_USER_DETAILS = '[Auth] Update User Details', - UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id' + UPDATE_LAST_PUBLIC_DASHBOARD_ID = '[Auth] Update Last Public Dashboard Id', + UPDATE_HAS_REPOSITORY = '[Auth] Change Has Repository' } export class ActionAuthAuthenticated implements Action { @@ -54,5 +55,11 @@ export class ActionAuthUpdateLastPublicDashboardId implements Action { constructor(readonly payload: { lastPublicDashboardId: string }) {} } +export class ActionAuthUpdateHasRepository implements Action { + readonly type = AuthActionTypes.UPDATE_HAS_REPOSITORY; + + constructor(readonly payload: { hasRepository: boolean }) {} +} + export type AuthActions = ActionAuthAuthenticated | ActionAuthUnauthenticated | - ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId; + ActionAuthLoadUser | ActionAuthUpdateUserDetails | ActionAuthUpdateLastPublicDashboardId | ActionAuthUpdateHasRepository; diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 9267d48e28..33b84945aa 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -20,6 +20,7 @@ export interface SysParamsState { userTokenAccessEnabled: boolean; allowedDashboardIds: string[]; edgesSupportEnabled: boolean; + hasRepository: boolean; } export interface AuthPayload extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.reducer.ts b/ui-ngx/src/app/core/auth/auth.reducer.ts index be2712300d..cace62660e 100644 --- a/ui-ngx/src/app/core/auth/auth.reducer.ts +++ b/ui-ngx/src/app/core/auth/auth.reducer.ts @@ -23,7 +23,8 @@ const emptyUserAuthState: AuthPayload = { userTokenAccessEnabled: false, forceFullscreen: false, allowedDashboardIds: [], - edgesSupportEnabled: false + edgesSupportEnabled: false, + hasRepository: false }; export const initialState: AuthState = { @@ -54,6 +55,9 @@ export function authReducer( case AuthActionTypes.UPDATE_LAST_PUBLIC_DASHBOARD_ID: return { ...state, ...action.payload}; + case AuthActionTypes.UPDATE_HAS_REPOSITORY: + return { ...state, ...action.payload}; + default: return state; } diff --git a/ui-ngx/src/app/core/auth/auth.selectors.ts b/ui-ngx/src/app/core/auth/auth.selectors.ts index aaadf00ec3..4a63dbcbed 100644 --- a/ui-ngx/src/app/core/auth/auth.selectors.ts +++ b/ui-ngx/src/app/core/auth/auth.selectors.ts @@ -55,6 +55,11 @@ export const selectUserTokenAccessEnabled = createSelector( (state: AuthState) => state.userTokenAccessEnabled ); +export const selectHasRepository = createSelector( + selectAuthState, + (state: AuthState) => state.hasRepository +); + export function getCurrentAuthState(store: Store): AuthState { let state: AuthState; store.pipe(select(selectAuth), take(1)).subscribe( diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index e26d7813da..c7b1a283d3 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -468,17 +468,27 @@ export class AuthService { return this.http.get('/api/edges/enabled', defaultHttpOptions()); } + private loadHasRepository(authUser: AuthUser): Observable { + if (authUser.authority === Authority.TENANT_ADMIN) { + return this.http.get('/api/admin/repositorySettings/exists', defaultHttpOptions()); + } else { + return of(false); + } + } + private loadSystemParams(authPayload: AuthPayload): Observable { const sources = [this.loadIsUserTokenAccessEnabled(authPayload.authUser), this.fetchAllowedDashboardIds(authPayload), this.loadIsEdgesSupportEnabled(), + this.loadHasRepository(authPayload.authUser), this.timeService.loadMaxDatapointsLimit()]; return forkJoin(sources) .pipe(map((data) => { const userTokenAccessEnabled: boolean = data[0] as boolean; const allowedDashboardIds: string[] = data[1] as string[]; const edgesSupportEnabled: boolean = data[2] as boolean; - return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled}; + const hasRepository: boolean = data[3] as boolean; + return {userTokenAccessEnabled, allowedDashboardIds, edgesSupportEnabled, hasRepository}; }, catchError((err) => { return of({}); }))); diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 8a12b998da..9b8c3b76b9 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -15,16 +15,21 @@ /// import { Injectable } from '@angular/core'; -import { defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; -import { Observable } from 'rxjs'; +import { defaultHttpOptions, defaultHttpOptionsFromConfig, RequestConfig } from './http-utils'; +import { Observable, of } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, + RepositorySettings, MailServerSettings, SecuritySettings, TestSmsRequest, - UpdateMessage + UpdateMessage, AutoCommitSettings } from '@shared/models/settings.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { tap } from 'rxjs/operators'; +import { AuthUser } from '@shared/models/user.model'; +import { Authority } from '@shared/models/authority.enum'; @Injectable({ providedIn: 'root' @@ -32,7 +37,8 @@ import { export class AdminService { constructor( - private http: HttpClient + private http: HttpClient, + private entitiesVersionControlService: EntitiesVersionControlService ) { } public getAdminSettings(key: string, config?: RequestConfig): Observable> { @@ -64,6 +70,50 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } + public getRepositorySettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveRepositorySettings(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings', repositorySettings, + defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); + } + + public deleteRepositorySettings(config?: RequestConfig) { + return this.http.delete('/api/admin/repositorySettings', defaultHttpOptionsFromConfig(config)).pipe( + tap(() => { + this.entitiesVersionControlService.clearBranchList(); + }) + ); + } + + public checkRepositoryAccess(repositorySettings: RepositorySettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/repositorySettings/checkAccess', repositorySettings, defaultHttpOptionsFromConfig(config)); + } + + public getAutoCommitSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/autoCommitSettings`, defaultHttpOptionsFromConfig(config)); + } + + public autoCommitSettingsExists(config?: RequestConfig): Observable { + return this.http.get('/api/admin/autoCommitSettings/exists', defaultHttpOptionsFromConfig(config)); + } + + public saveAutoCommitSettings(autoCommitSettings: AutoCommitSettings, + config?: RequestConfig): Observable { + return this.http.post('/api/admin/autoCommitSettings', autoCommitSettings, defaultHttpOptionsFromConfig(config)); + } + + public deleteAutoCommitSettings(config?: RequestConfig) { + return this.http.delete('/api/admin/autoCommitSettings', defaultHttpOptionsFromConfig(config)); + } + public checkUpdates(config?: RequestConfig): Observable { return this.http.get(`/api/admin/updates`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts new file mode 100644 index 0000000000..d7e892f1db --- /dev/null +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -0,0 +1,190 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Injectable } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { defaultHttpOptionsFromConfig, RequestConfig } from '@core/http/http-utils'; +import { Observable, of, timer } from 'rxjs'; +import { + BranchInfo, + EntityDataDiff, + EntityDataInfo, + EntityLoadError, + entityLoadErrorTranslationMap, + EntityLoadErrorType, + EntityVersion, + VersionCreateRequest, + VersionCreationResult, + VersionLoadRequest, + VersionLoadResult +} from '@shared/models/vc.models'; +import { PageLink } from '@shared/models/page/page-link'; +import { PageData } from '@shared/models/page/page-data'; +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectIsUserLoaded } from '@core/auth/auth.selectors'; +import { catchError, finalize, map, switchMap, takeWhile, tap } from 'rxjs/operators'; +import { TranslateService } from '@ngx-translate/core'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { ActionLoadFinish, ActionLoadStart } from '@core/interceptors/load.actions'; +import { NULL_UUID } from '@shared/models/id/has-uuid'; + +@Injectable({ + providedIn: 'root' +}) +export class EntitiesVersionControlService { + + branchList: Array = null; + + constructor( + private http: HttpClient, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private store: Store + ) { + + this.store.pipe(select(selectIsUserLoaded)).subscribe( + () => { + this.branchList = null; + } + ); + } + + public clearBranchList(): void { + this.branchList = null; + } + + public listBranches(): Observable> { + if (!this.branchList) { + return this.http.get>('/api/entities/vc/branches', + defaultHttpOptionsFromConfig({ignoreErrors: true, ignoreLoading: false})).pipe( + catchError(() => of([] as Array)), + tap((list) => { + this.branchList = list; + }) + ); + } else { + return of(this.branchList); + } + } + + public getEntityDataInfo(externalEntityId: EntityId, + versionId: string, + config?: RequestConfig): Observable { + return this.http.get(`/api/entities/vc/info/${versionId}/${externalEntityId.entityType}/${externalEntityId.id}`, + defaultHttpOptionsFromConfig(config)); + } + + public saveEntitiesVersion(request: VersionCreateRequest, config?: RequestConfig): Observable { + this.store.dispatch(new ActionLoadStart()); + return this.http.post('/api/entities/vc/version', request, + defaultHttpOptionsFromConfig({...config, ...{ignoreLoading: true}})).pipe( + switchMap((requestId) => { + return timer(0, 2000).pipe( + switchMap(() => this.getVersionCreateRequestStatus(requestId, config)), + takeWhile((res) => !res.done, true) + ); + }), + finalize(() => { + const branch = request.branch; + if (this.branchList && !this.branchList.find(b => b.name === branch)) { + this.branchList = null; + } + this.store.dispatch(new ActionLoadFinish()); + }), + ); + } + + private getVersionCreateRequestStatus(requestId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/entities/vc/version/${requestId}/status`, + defaultHttpOptionsFromConfig({...config, ...{ignoreLoading: true}})); + } + + public listEntityVersions(pageLink: PageLink, branch: string, + externalEntityId: EntityId, + config?: RequestConfig): Observable> { + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}&branch=${encodedBranch}`, + defaultHttpOptionsFromConfig(config)); + } + + public listEntityTypeVersions(pageLink: PageLink, branch: string, + entityType: EntityType, + config?: RequestConfig): Observable> { + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version/${entityType}${pageLink.toQuery()}&branch=${encodedBranch}`, + defaultHttpOptionsFromConfig(config)); + } + + public listVersions(pageLink: PageLink, branch: string, + config?: RequestConfig): Observable> { + const encodedBranch = encodeURIComponent(branch); + return this.http.get>(`/api/entities/vc/version${pageLink.toQuery()}&branch=${encodedBranch}`, + defaultHttpOptionsFromConfig(config)); + } + + public loadEntitiesVersion(request: VersionLoadRequest, config?: RequestConfig): Observable { + this.store.dispatch(new ActionLoadStart()); + return this.http.post('/api/entities/vc/entity', request, + defaultHttpOptionsFromConfig({...config, ...{ignoreLoading: true}})).pipe( + switchMap((requestId) => { + return timer(0, 2000).pipe( + switchMap(() => this.getVersionLoadRequestStatus(requestId, config)), + takeWhile((res) => !res.done, true), + ); + }), + finalize(() => { + this.store.dispatch(new ActionLoadFinish()); + }), + ); + } + + private getVersionLoadRequestStatus(requestId: string, config?: RequestConfig): Observable { + return this.http.get(`/api/entities/vc/entity/${requestId}/status`, + defaultHttpOptionsFromConfig({...config, ...{ignoreLoading: true}})); + } + + public compareEntityDataToVersion(branch: string, + entityId: EntityId, + versionId: string, + config?: RequestConfig): Observable { + const encodedBranch = encodeURIComponent(branch); + return this.http.get(`/api/entities/vc/diff/${entityId.entityType}/${entityId.id}?branch=${encodedBranch}&versionId=${versionId}`, + defaultHttpOptionsFromConfig(config)); + } + + public entityLoadErrorToMessage(entityLoadError: EntityLoadError): SafeHtml { + const type = entityLoadError.type; + const messageId = entityLoadErrorTranslationMap.get(type); + const messageArgs = {} as any; + switch (type) { + case EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT: + messageArgs.entityId = entityLoadError.source.id; + break; + case EntityLoadErrorType.MISSING_REFERENCED_ENTITY: + messageArgs.sourceEntityTypeName = + (this.translate.instant(entityTypeTranslations.get(entityLoadError.source.entityType).type) as string).toLowerCase(); + messageArgs.sourceEntityId = entityLoadError.source.id; + messageArgs.targetEntityTypeName = + (this.translate.instant(entityTypeTranslations.get(entityLoadError.target.entityType).type) as string).toLowerCase(); + messageArgs.targetEntityId = entityLoadError.target.id; + break; + } + return this.sanitizer.bypassSecurityTrustHtml(this.translate.instant(messageId, messageArgs)); + } +} diff --git a/ui-ngx/src/app/core/http/entity.service.ts b/ui-ngx/src/app/core/http/entity.service.ts index 1d20b631ae..b1c395c2ba 100644 --- a/ui-ngx/src/app/core/http/entity.service.ts +++ b/ui-ngx/src/app/core/http/entity.service.ts @@ -67,7 +67,6 @@ import { AlarmData, AlarmDataQuery, createDefaultEntityDataPageLink, - defaultEntityDataPageLink, EntityData, EntityDataQuery, entityDataToEntityInfo, @@ -233,6 +232,11 @@ export class EntityService { case EntityType.ALARM: console.error('Get Alarm Entity is not implemented!'); break; + case EntityType.DEVICE_PROFILE: + observable = this.getEntitiesByIdsObservable( + (id) => this.deviceProfileService.getDeviceProfileInfo(id, config), + entityIds); + break; } return observable; } @@ -374,6 +378,10 @@ export class EntityService { pageLink.sortOrder.property = 'title'; entitiesObservable = this.otaPackageService.getOtaPackages(pageLink, config); break; + case EntityType.DEVICE_PROFILE: + pageLink.sortOrder.property = 'name'; + entitiesObservable = this.deviceProfileService.getDeviceProfileInfos(pageLink, null, config); + break; } return entitiesObservable; } diff --git a/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts b/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts index 94968f393a..15c4d3ede3 100644 --- a/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts +++ b/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts @@ -29,6 +29,7 @@ import { ActionLoadFinish, ActionLoadStart } from './load.actions'; import { ActionNotificationShow } from '@app/core/notification/notification.actions'; import { DialogService } from '@core/services/dialog.service'; import { TranslateService } from '@ngx-translate/core'; +import { parseHttpErrorMessage } from '@core/utils'; let tmpHeaders = {}; @@ -131,43 +132,12 @@ export class GlobalHttpInterceptor implements HttpInterceptor { } if (unhandled && !ignoreErrors) { - let error = null; - if (req.responseType === 'text') { - try { - error = errorResponse.error ? JSON.parse(errorResponse.error) : null; - } catch (e) {} - } else { - error = errorResponse.error; - } - if (error && !error.message) { - this.showError(this.prepareMessageFromData(error)); - } else if (error && error.message) { - this.showError(error.message, error.timeout ? error.timeout : 0); - } else { - this.showError('Unhandled error code ' + (error ? error.status : '\'Unknown\'')); - } + const errorMessageWithTimeout = parseHttpErrorMessage(errorResponse, this.translate, req.responseType); + this.showError(errorMessageWithTimeout.message, errorMessageWithTimeout.timeout); } return throwError(errorResponse); } - private prepareMessageFromData(data) { - if (typeof data === 'object' && data.constructor === ArrayBuffer) { - const msg = String.fromCharCode.apply(null, new Uint8Array(data)); - try { - const msgObj = JSON.parse(msg); - if (msgObj.message) { - return msgObj.message; - } else { - return msg; - } - } catch (e) { - return msg; - } - } else { - return data; - } - } - private retryRequest(req: HttpRequest, next: HttpHandler): Observable> { const thisTimeout = 1000 + Math.random() * 3000; return of(null).pipe( diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index 78c791ba10..ffc4884b7f 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -357,6 +357,13 @@ export class MenuService { path: '/dashboards', icon: 'dashboards' }, + { + id: guid(), + name: 'version-control.version-control', + type: 'link', + path: '/vc', + icon: 'history' + }, { id: guid(), name: 'audit-log.audit-logs', @@ -376,7 +383,7 @@ export class MenuService { name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '80px', + height: '160px', icon: 'settings', pages: [ { @@ -392,6 +399,20 @@ export class MenuService { type: 'link', path: '/settings/resources-library', icon: 'folder' + }, + { + id: guid(), + name: 'admin.repository-settings', + type: 'link', + path: '/settings/repository', + icon: 'manage_history' + }, + { + id: guid(), + name: 'admin.auto-commit-settings', + type: 'link', + path: '/settings/auto-commit', + icon: 'settings_backup_restore' } ] } @@ -499,6 +520,16 @@ export class MenuService { } ] }, + { + name: 'version-control.management', + places: [ + { + name: 'version-control.version-control', + icon: 'history', + path: '/vc' + } + ] + }, { name: 'audit-log.audit', places: [ @@ -526,6 +557,16 @@ export class MenuService { name: 'resource.resources-library', icon: 'folder', path: '/settings/resources-library' + }, + { + name: 'admin.repository-settings', + icon: 'manage_history', + path: '/settings/repository', + }, + { + name: 'admin.auto-commit-settings', + icon: 'settings_backup_restore', + path: '/settings/auto-commit' } ] } diff --git a/ui-ngx/src/app/core/services/script/node-script-test.service.ts b/ui-ngx/src/app/core/services/script/node-script-test.service.ts index 7428bc40f5..5dcdb192af 100644 --- a/ui-ngx/src/app/core/services/script/node-script-test.service.ts +++ b/ui-ngx/src/app/core/services/script/node-script-test.service.ts @@ -85,7 +85,7 @@ export class NodeScriptTestService { return this.dialog.open(NodeScriptTestDialogComponent, { disableClose: true, - panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-sm'], + panelClass: ['tb-dialog', 'tb-fullscreen-dialog', 'tb-fullscreen-dialog-gt-xs'], data: { msg, metadata, diff --git a/ui-ngx/src/app/core/services/utils.service.ts b/ui-ngx/src/app/core/services/utils.service.ts index fcef76f612..b901b87328 100644 --- a/ui-ngx/src/app/core/services/utils.service.ts +++ b/ui-ngx/src/app/core/services/utils.service.ts @@ -284,6 +284,11 @@ export class UtilsService { if (!datasource.dataKeys) { datasource.dataKeys = []; } + datasource.dataKeys.forEach(dataKey => { + if (isUndefined(dataKey.label)) { + dataKey.label = dataKey.name; + } + }); }); return datasources; } diff --git a/ui-ngx/src/app/core/utils.ts b/ui-ngx/src/app/core/utils.ts index 37d2f1f652..bedc963186 100644 --- a/ui-ngx/src/app/core/utils.ts +++ b/ui-ngx/src/app/core/utils.ts @@ -21,6 +21,10 @@ import { Datasource, DatasourceData, FormattedData, ReplaceInfo } from '@app/sha import { EntityId } from '@shared/models/id/entity-id'; import { NULL_UUID } from '@shared/models/id/has-uuid'; import { EntityType, baseDetailsPageByEntityType } from '@shared/models/entity-type.models'; +import { HttpErrorResponse } from '@angular/common/http'; +import { letterSpacing } from 'html2canvas/dist/types/css/property-descriptors/letter-spacing'; +import { TranslateService } from '@ngx-translate/core'; +import { serverErrorCodesTranslations } from '@shared/models/constants'; const varsRegex = /\${([^}]*)}/g; @@ -96,7 +100,7 @@ export function isEmptyStr(value: any): boolean { } export function isNotEmptyStr(value: any): boolean { - return value !== null && typeof value === 'string' && value.trim().length > 0; + return typeof value === 'string' && value.trim().length > 0; } export function isFunction(value: any): boolean { @@ -664,3 +668,53 @@ export function randomAlphanumeric(length: number): string { export function getEntityDetailsPageURL(id: string, entityType: EntityType): string { return `${baseDetailsPageByEntityType.get(entityType)}/${id}`; } + +export function parseHttpErrorMessage(errorResponse: HttpErrorResponse, + translate: TranslateService, responseType?: string): {message: string, timeout: number} { + let error = null; + let errorMessage: string; + let timeout = 0; + if (responseType === 'text') { + try { + error = errorResponse.error ? JSON.parse(errorResponse.error) : null; + } catch (e) {} + } else { + error = errorResponse.error; + } + if (error && !error.message) { + errorMessage = prepareMessageFromData(error); + } else if (error && error.message) { + errorMessage = error.message; + timeout = error.timeout ? error.timeout : 0; + } else { + errorMessage = `Unhandled error code ${error ? error.status : '\'Unknown\''}`; + } + if (isObject(errorMessage)) { + let errorText = `${errorResponse.status}: `; + let errorKey = null; + if ((errorMessage as any).errorCode) { + errorKey = serverErrorCodesTranslations.get((errorMessage as any).errorCode); + } + errorText += errorKey ? translate.instant(errorKey) : errorResponse.statusText; + errorMessage = errorText; + } + return {message: errorMessage, timeout}; +} + +function prepareMessageFromData(data): string { + if (typeof data === 'object' && data.constructor === ArrayBuffer) { + const msg = String.fromCharCode.apply(null, new Uint8Array(data)); + try { + const msgObj = JSON.parse(msg); + if (msgObj.message) { + return msgObj.message; + } else { + return msg; + } + } catch (e) { + return msg; + } + } else { + return data; + } +} diff --git a/ui-ngx/src/app/modules/home/components/alarm/alarm-details-dialog.component.ts b/ui-ngx/src/app/modules/home/components/alarm/alarm-details-dialog.component.ts index dc88580177..be3aff8c50 100644 --- a/ui-ngx/src/app/modules/home/components/alarm/alarm-details-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/alarm/alarm-details-dialog.component.ts @@ -35,7 +35,8 @@ import { DatePipe } from '@angular/common'; import { TranslateService } from '@ngx-translate/core'; export interface AlarmDetailsDialogData { - alarmId: string; + alarmId?: string; + alarm?: AlarmInfo; allowAcknowledgment: boolean; allowClear: boolean; displayDetails: boolean; @@ -48,6 +49,7 @@ export interface AlarmDetailsDialogData { }) export class AlarmDetailsDialogComponent extends DialogComponent implements OnInit { + alarmId: string; alarmFormGroup: FormGroup; allowAcknowledgment: boolean; @@ -93,12 +95,17 @@ export class AlarmDetailsDialogComponent extends DialogComponent this.loadAlarmSubject.next(alarm) ); } @@ -140,15 +147,25 @@ export class AlarmDetailsDialogComponent extends DialogComponent { this.alarmUpdated = true; this.loadAlarm(); } - ); + if (this.alarmId) { + this.alarmService.ackAlarm(this.alarmId).subscribe( + () => { + this.alarmUpdated = true; + this.loadAlarm(); + } + ); + } } clear(): void { - this.alarmService.clearAlarm(this.data.alarmId).subscribe( - () => { this.alarmUpdated = true; this.loadAlarm(); } - ); + if (this.alarmId) { + this.alarmService.clearAlarm(this.alarmId).subscribe( + () => { + this.alarmUpdated = true; + this.loadAlarm(); + } + ); + } } } diff --git a/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts b/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts index d35f82daf8..6cc51857e7 100644 --- a/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts @@ -44,9 +44,15 @@ import { AlarmDetailsDialogData } from '@home/components/alarm/alarm-details-dialog.component'; import { DAY, historyInterval } from '@shared/models/time/time.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { Authority } from '@shared/models/authority.enum'; export class AlarmTableConfig extends EntityTableConfig { + private authUser = getCurrentAuthUser(this.store); + searchStatus: AlarmSearchStatus; constructor(private alarmService: AlarmService, @@ -55,7 +61,8 @@ export class AlarmTableConfig extends EntityTableConfig private datePipe: DatePipe, private dialog: MatDialog, public entityId: EntityId = null, - private defaultSearchStatus: AlarmSearchStatus = AlarmSearchStatus.ANY) { + private defaultSearchStatus: AlarmSearchStatus = AlarmSearchStatus.ANY, + private store: Store) { super(); this.loadDataOnInit = false; this.tableTitle = ''; @@ -114,6 +121,7 @@ export class AlarmTableConfig extends EntityTableConfig } showAlarmDetails(entity: AlarmInfo) { + const isPermissionWrite = this.authUser.authority !== Authority.CUSTOMER_USER || entity.customerId.id === this.authUser.customerId; this.dialog.open (AlarmDetailsDialogComponent, { @@ -121,8 +129,9 @@ export class AlarmTableConfig extends EntityTableConfig panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { alarmId: entity.id.id, - allowAcknowledgment: true, - allowClear: true, + alarm: entity, + allowAcknowledgment: isPermissionWrite, + allowClear: isPermissionWrite, displayDetails: true } }).afterClosed().subscribe( diff --git a/ui-ngx/src/app/modules/home/components/alarm/alarm-table.component.ts b/ui-ngx/src/app/modules/home/components/alarm/alarm-table.component.ts index cf8c66c93a..d642bcef7a 100644 --- a/ui-ngx/src/app/modules/home/components/alarm/alarm-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/alarm/alarm-table.component.ts @@ -24,6 +24,8 @@ import { DialogService } from '@core/services/dialog.service'; import { AlarmTableConfig } from './alarm-table-config'; import { AlarmSearchStatus } from '@shared/models/alarm.models'; import { AlarmService } from '@app/core/http/alarm.service'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; @Component({ selector: 'tb-alarm-table', @@ -68,7 +70,8 @@ export class AlarmTableComponent implements OnInit { private dialogService: DialogService, private translate: TranslateService, private datePipe: DatePipe, - private dialog: MatDialog) { + private dialog: MatDialog, + private store: Store) { } ngOnInit() { @@ -80,7 +83,8 @@ export class AlarmTableComponent implements OnInit { this.datePipe, this.dialog, this.entityIdValue, - AlarmSearchStatus.ANY + AlarmSearchStatus.ANY, + this.store ); } diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts index 6ff3f78e7d..9d6812e0f9 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts @@ -285,6 +285,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI this.attributeScopeSelectionReadonly = true; } this.mode = 'default'; + this.textSearchMode = false; this.selectedWidgetsBundleAlias = null; this.attributeScope = this.defaultAttributeScope; this.pageLink.textSearch = null; diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index dd7212cf5b..4caa27b0a2 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -81,6 +81,13 @@ (click)="isFullscreen = !isFullscreen"> {{ isFullscreen ? 'fullscreen_exit' : 'fullscreen' }} + + + + +
+ +
+
+ + +
+ + +
+
+
+ + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
+
+
+ + + + +
+ admin.no-auto-commit-entities-prompt +
+
+ + + +
+ + +
+ + + +
+ + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss new file mode 100644 index 0000000000..ea8017b9f1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.scss @@ -0,0 +1,74 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.auto-commit-settings { + margin: 8px; + .mat-divider { + position: relative; + } + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + tb-branch-autocomplete { + min-width: 200px; + max-width: 200px; + display: block; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} + diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts new file mode 100644 index 0000000000..fd100f6247 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.ts @@ -0,0 +1,214 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { AbstractControl, FormArray, FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { AutoCommitSettings, AutoVersionCreateConfig } from '@shared/models/settings.models'; +import { TranslateService } from '@ngx-translate/core'; +import { DialogService } from '@core/services/dialog.service'; +import { catchError, mergeMap } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { EntityTypeVersionCreateConfig, exportableEntityTypes } from '@shared/models/vc.models'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-auto-commit-settings', + templateUrl: './auto-commit-settings.component.html', + styleUrls: ['./auto-commit-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class AutoCommitSettingsComponent extends PageComponent implements OnInit { + + autoCommitSettingsForm: FormGroup; + settings: AutoCommitSettings = null; + + entityTypes = EntityType; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private sanitizer: DomSanitizer, + private translate: TranslateService, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.autoCommitSettingsForm = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.adminService.autoCommitSettingsExists().pipe( + catchError(() => of(false)), + mergeMap((hasAutoCommitSettings) => { + if (hasAutoCommitSettings) { + return this.adminService.getAutoCommitSettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(settings), {emitEvent: false}); + }); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.autoCommitSettingsForm.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.autoCommitSettingsForm.get('entityTypes') as FormArray).removeAt(index); + this.autoCommitSettingsForm.markAsDirty(); + } + + public addEnabled(): boolean { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + const config: AutoVersionCreateConfig = { + branch: null, + saveAttributes: true, + saveRelations: false, + saveCredentials: true + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + public removeAll() { + const entityTypesArray = this.autoCommitSettingsForm.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.autoCommitSettingsForm.updateValueAndValidity(); + this.autoCommitSettingsForm.markAsDirty(); + } + + entityTypeText(entityTypeControl: AbstractControl): SafeHtml { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: AutoVersionCreateConfig = entityTypeControl.get('config').value; + let message = entityType ? this.translate.instant(entityTypeTranslations.get(entityType).typePlural) : 'Undefined'; + let branchName; + if (config.branch) { + branchName = config.branch; + } else { + branchName = this.translate.instant('version-control.default'); + } + message += ` (${this.translate.instant('version-control.auto-commit-to-branch', {branch: branchName})})`; + return this.sanitizer.bypassSecurityTrustHtml(message); + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + save(): void { + const value: [{entityType: string, config: AutoVersionCreateConfig}] = + this.autoCommitSettingsForm.get('entityTypes').value || []; + const settings: AutoCommitSettings = {}; + if (value && value.length) { + value.forEach((val) => { + settings[val.entityType] = val.config; + }); + } + this.adminService.saveAutoCommitSettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(savedSettings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-auto-commit-settings-title', ), + this.translate.instant('admin.delete-auto-commit-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteAutoCommitSettings().subscribe( + () => { + this.settings = null; + this.autoCommitSettingsForm.setControl('entityTypes', + this.prepareEntityTypesFormArray(this.settings), {emitEvent: false}); + this.autoCommitSettingsForm.markAsPristine(); + } + ); + } + }); + } + + private prepareEntityTypesFormArray(settings: AutoCommitSettings | null): FormArray { + const entityTypesControls: Array = []; + if (settings) { + for (const entityType of Object.keys(settings)) { + const config = settings[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: AutoVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + branch: [config.branch, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []] + }) + } + ); + return entityTypeControl; + } + + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html new file mode 100644 index 0000000000..3e849773c3 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.html @@ -0,0 +1,93 @@ + +
+
+ +

{{ 'version-control.create-entities-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + version-control.default-sync-strategy + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + + + + +
+
+
+
+ + +
+
+
+
+
+
+
+ +
+ +
+
+ version-control.creating-version + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts new file mode 100644 index 0000000000..5348b4b340 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -0,0 +1,144 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnDestroy, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + ComplexVersionCreateRequest, + createDefaultEntityTypesVersionCreate, + SyncStrategy, syncStrategyHintMap, syncStrategyTranslationMap, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { Observable, Subscription } from 'rxjs'; +import { share } from 'rxjs/operators'; +import { parseHttpErrorMessage } from '@core/utils'; + +@Component({ + selector: 'tb-complex-version-create', + templateUrl: './complex-version-create.component.html', + styleUrls: ['./version-control.scss'] +}) +export class ComplexVersionCreateComponent extends PageComponent implements OnInit, OnDestroy { + + @Input() + branch: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + popoverComponent: TbPopoverComponent; + + createVersionFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + syncStrategyHints = syncStrategyHintMap; + + resultMessage: SafeHtml; + + hasError = false; + + versionCreateResult: VersionCreationResult = null; + + versionCreateBranch: string = null; + + versionCreateResult$: Observable; + + private versionCreateResultSubscription: Subscription; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private sanitizer: DomSanitizer, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [null, [Validators.required]], + syncStrategy: [SyncStrategy.MERGE, Validators.required], + entityTypes: [createDefaultEntityTypesVersionCreate(), []], + }); + } + + ngOnDestroy() { + super.ngOnDestroy(); + if (this.versionCreateResultSubscription) { + this.versionCreateResultSubscription.unsubscribe(); + } + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionCreateResult, this.versionCreateBranch); + } + } + + export(): void { + const request: ComplexVersionCreateRequest = { + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + syncStrategy: this.createVersionFormGroup.get('syncStrategy').value, + entityTypes: this.createVersionFormGroup.get('entityTypes').value, + type: VersionCreateRequestType.COMPLEX + }; + + this.versionCreateResult$ = this.entitiesVersionControlService.saveEntitiesVersion(request, {ignoreErrors: true}).pipe( + share() + ); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + + this.versionCreateResultSubscription = this.versionCreateResult$.subscribe((result) => { + if (result.done && !result.added && !result.modified && !result.removed) { + this.resultMessage = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.nothing-to-commit')); + } else { + this.resultMessage = this.sanitizer.bypassSecurityTrustHtml(result.error ? result.error : this.translate.instant('version-control.version-create-result', + {added: result.added, modified: result.modified, removed: result.removed})); + } + this.versionCreateResult = result; + this.versionCreateBranch = request.branch; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }, + (error) => { + this.hasError = true; + this.resultMessage = this.sanitizer.bypassSecurityTrustHtml(parseHttpErrorMessage(error, this.translate).message); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html new file mode 100644 index 0000000000..05f16b3e2a --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -0,0 +1,74 @@ + +
+
+ +

{{ 'version-control.restore-entities-from-version' | translate: {versionName} }}

+ + + + +
+
+
+ + +
+
+
+
+ + +
+
+
+
+ {{ 'version-control.no-entities-restored' | translate }} +
+
+
{{ entityTypeLoadResultMessage(entityTypeLoadResult) }}
+
+ +
+ +
+
+ version-control.restoring-entities-from-version + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts new file mode 100644 index 0000000000..718dc5c2e7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.ts @@ -0,0 +1,152 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnDestroy, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + createDefaultEntityTypesVersionLoad, EntityTypeLoadResult, + EntityTypeVersionLoadRequest, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { TranslateService } from '@ngx-translate/core'; +import { entityTypeTranslations } from '@shared/models/entity-type.models'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { Observable, Subscription } from 'rxjs'; +import { share } from 'rxjs/operators'; +import { parseHttpErrorMessage } from '@core/utils'; + +@Component({ + selector: 'tb-complex-version-load', + templateUrl: './complex-version-load.component.html', + styleUrls: ['./version-control.scss'] +}) +export class ComplexVersionLoadComponent extends PageComponent implements OnInit, OnDestroy { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + onClose: (result: VersionLoadResult | null) => void; + + @Input() + popoverComponent: TbPopoverComponent; + + loadVersionFormGroup: FormGroup; + + versionLoadResult: VersionLoadResult = null; + + entityTypeLoadResults: Array = null; + + errorMessage: SafeHtml; + + hasError = false; + + versionLoadResult$: Observable; + + private versionLoadResultSubscription: Subscription; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.loadVersionFormGroup = this.fb.group({ + entityTypes: [createDefaultEntityTypesVersionLoad(), []], + }); + } + + ngOnDestroy() { + super.ngOnDestroy(); + if (this.versionLoadResultSubscription) { + this.versionLoadResultSubscription.unsubscribe(); + } + } + + entityTypeLoadResultMessage(result: EntityTypeLoadResult): string { + const entityType = result.entityType; + let message = this.translate.instant(entityTypeTranslations.get(entityType).typePlural) + ': '; + const resultMessages: string[] = []; + if (result.created) { + resultMessages.push(this.translate.instant('version-control.created', {created: result.created})); + } + if (result.updated) { + resultMessages.push(this.translate.instant('version-control.updated', {updated: result.updated})); + } + if (result.deleted) { + resultMessages.push(this.translate.instant('version-control.deleted', {deleted: result.deleted})); + } + message += resultMessages.join(', ') + '.'; + return message; + } + + cancel(): void { + if (this.onClose) { + this.onClose(this.versionLoadResult); + } + } + + restore(): void { + const request: EntityTypeVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + entityTypes: this.loadVersionFormGroup.get('entityTypes').value, + type: VersionLoadRequestType.ENTITY_TYPE + }; + this.versionLoadResult$ = this.entitiesVersionControlService.loadEntitiesVersion(request, {ignoreErrors: true}).pipe( + share() + ); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + this.versionLoadResultSubscription = this.versionLoadResult$.subscribe((result) => { + this.versionLoadResult = result; + this.entityTypeLoadResults = (result.result || []).filter(res => res.created || res.updated || res.deleted); + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + } + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }, + (error) => { + this.hasError = true; + this.errorMessage = this.sanitizer.bypassSecurityTrustHtml(parseHttpErrorMessage(error, this.translate).message); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html new file mode 100644 index 0000000000..928b7a7668 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -0,0 +1,119 @@ + +
+
+ version-control.entities-to-export +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+ + version-control.sync-strategy + + + {{ 'version-control.default' | translate }} + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + +
+ + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
+
+
+
+ + {{ 'version-control.all-entities' | translate }} + + + +
+
+ + +
+
+
+ version-control.no-entities-to-export-prompt +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss new file mode 100644 index 0000000000..f90ad2a803 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-create { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts new file mode 100644 index 0000000000..6434329447 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.ts @@ -0,0 +1,253 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { + EntityTypeVersionCreateConfig, + exportableEntityTypes, + SyncStrategy, + syncStrategyTranslationMap +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { isDefinedAndNotNull } from '@core/utils'; + +@Component({ + selector: 'tb-entity-types-version-create', + templateUrl: './entity-types-version-create.component.html', + styleUrls: ['./entity-types-version-create.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionCreateComponent), + multi: true + } + ] +}) +export class EntityTypesVersionCreateComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionCreateConfig}; + + private propagateChange = null; + + public entityTypesVersionCreateFormGroup: FormGroup; + + syncStrategies = Object.values(SyncStrategy); + + syncStrategyTranslations = syncStrategyTranslationMap; + + entityTypes = EntityType; + + constructor(protected store: Store, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionCreateFormGroup = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.entityTypesVersionCreateFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionCreateFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionCreateFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionCreateFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionCreateFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionCreateConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + syncStrategy: [config.syncStrategy === null ? 'default' : config.syncStrategy, []], + saveRelations: [config.saveRelations, []], + saveAttributes: [config.saveAttributes, []], + saveCredentials: [config.saveCredentials, []], + allEntities: [config.allEntities, []], + entityIds: [config.entityIds, [Validators.required]] + }) + } + ); + this.updateEntityTypeValidators(entityTypeControl); + entityTypeControl.get('config').get('allEntities').valueChanges.subscribe(() => { + this.updateEntityTypeValidators(entityTypeControl); + }); + return entityTypeControl; + } + + private updateEntityTypeValidators(entityTypeControl: AbstractControl): void { + const allEntities: boolean = entityTypeControl.get('config').get('allEntities').value; + if (allEntities) { + entityTypeControl.get('config').get('entityIds').disable({emitEvent: false}); + } else { + entityTypeControl.get('config').get('entityIds').enable({emitEvent: false}); + } + entityTypeControl.get('config').get('entityIds').updateValueAndValidity({emitEvent: false}); + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionCreateConfig = { + syncStrategy: null, + saveAttributes: true, + saveRelations: true, + saveCredentials: true, + allEntities: true, + entityIds: [] + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionCreateFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionCreateFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + const config: EntityTypeVersionCreateConfig = entityTypeControl.get('config').value; + let count = config?.entityIds?.length; + if (!isDefinedAndNotNull(count)) { + count = 0; + } + if (entityType) { + return this.translate.instant((config?.allEntities ? entityTypeTranslations.get(entityType).typePlural + : entityTypeTranslations.get(entityType).list), { count }); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionCreateConfig}] = + this.entityTypesVersionCreateFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionCreateConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + if ((modelValue[val.entityType].syncStrategy as any) === 'default') { + modelValue[val.entityType].syncStrategy = null; + } + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html new file mode 100644 index 0000000000..bb3c511e0c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.html @@ -0,0 +1,105 @@ + +
+
+ version-control.entities-to-restore +
+
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.find-existing-entity-by-name' | translate }} + +
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+ version-control.no-entities-to-restore-prompt +
+
+ + + +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss new file mode 100644 index 0000000000..b3e840198c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.scss @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .entity-types-version-load { + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } + + .tb-control-list { + overflow-y: auto; + max-height: 600px; + } + + .tb-prompt { + margin: 30px 0; + } + + mat-expansion-panel.entity-type-config { + box-shadow: none; + border: 1px groove rgba(0, 0, 0, .25); + .mat-expansion-panel-header { + padding: 0 24px 0 8px; + height: 48px; + } + .entity-type-config-content { + padding: 0 8px 8px; + } + } + } +} + +:host ::ng-deep { + .mat-expansion-panel.entity-type-config { + .mat-expansion-panel-body { + padding: 0; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts new file mode 100644 index 0000000000..ef43899c5b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-load.component.ts @@ -0,0 +1,248 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnInit, Renderer2, ViewContainerRef } from '@angular/core'; +import { + AbstractControl, + ControlValueAccessor, + FormArray, + FormBuilder, + FormControl, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + Validator, + Validators +} from '@angular/forms'; +import { PageComponent } from '@shared/components/page.component'; +import { EntityTypeVersionLoadConfig, exportableEntityTypes, VersionCreationResult } from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.models'; +import { MatCheckbox } from '@angular/material/checkbox/checkbox'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { RemoveOtherEntitiesConfirmComponent } from '@home/components/vc/remove-other-entities-confirm.component'; + +@Component({ + selector: 'tb-entity-types-version-load', + templateUrl: './entity-types-version-load.component.html', + styleUrls: ['./entity-types-version-load.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => EntityTypesVersionLoadComponent), + multi: true + } + ] +}) +export class EntityTypesVersionLoadComponent extends PageComponent implements OnInit, ControlValueAccessor, Validator { + + @Input() + disabled: boolean; + + private modelValue: {[entityType: string]: EntityTypeVersionLoadConfig}; + + private propagateChange = null; + + public entityTypesVersionLoadFormGroup: FormGroup; + + entityTypes = EntityType; + + constructor(protected store: Store, + private translate: TranslateService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private viewContainerRef: ViewContainerRef, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.entityTypesVersionLoadFormGroup = this.fb.group({ + entityTypes: this.fb.array([], []) + }); + this.entityTypesVersionLoadFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (isDisabled) { + this.entityTypesVersionLoadFormGroup.disable({emitEvent: false}); + } else { + this.entityTypesVersionLoadFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): void { + this.modelValue = value; + this.entityTypesVersionLoadFormGroup.setControl('entityTypes', + this.prepareEntityTypesFormArray(value), {emitEvent: false}); + } + + public validate(c: FormControl) { + return this.entityTypesVersionLoadFormGroup.valid && this.entityTypesFormGroupArray().length ? null : { + entityTypes: { + valid: false, + }, + }; + } + + private prepareEntityTypesFormArray(entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig} | undefined): FormArray { + const entityTypesControls: Array = []; + if (entityTypes) { + for (const entityType of Object.keys(entityTypes)) { + const config = entityTypes[entityType]; + entityTypesControls.push(this.createEntityTypeControl(entityType as EntityType, config)); + } + } + return this.fb.array(entityTypesControls); + } + + private createEntityTypeControl(entityType: EntityType, config: EntityTypeVersionLoadConfig): AbstractControl { + const entityTypeControl = this.fb.group( + { + entityType: [entityType, [Validators.required]], + config: this.fb.group({ + loadRelations: [config.loadRelations, []], + loadAttributes: [config.loadAttributes, []], + loadCredentials: [config.loadCredentials, []], + removeOtherEntities: [config.removeOtherEntities, []], + findExistingEntityByName: [config.findExistingEntityByName, []] + }) + } + ); + return entityTypeControl; + } + + entityTypesFormGroupArray(): FormGroup[] { + return (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).controls as FormGroup[]; + } + + entityTypesFormGroupExpanded(entityTypeControl: AbstractControl): boolean { + return !!(entityTypeControl as any).expanded; + } + + public trackByEntityType(index: number, entityTypeControl: AbstractControl): any { + return entityTypeControl; + } + + public removeEntityType(index: number) { + (this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray).removeAt(index); + } + + public addEnabled(): boolean { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + return entityTypesArray.length < exportableEntityTypes.length; + } + + public addEntityType() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + const config: EntityTypeVersionLoadConfig = { + loadAttributes: true, + loadRelations: true, + loadCredentials: true, + removeOtherEntities: false, + findExistingEntityByName: true + }; + const allowed = this.allowedEntityTypes(); + let entityType: EntityType = null; + if (allowed.length) { + entityType = allowed[0]; + } + const entityTypeControl = this.createEntityTypeControl(entityType, config); + (entityTypeControl as any).expanded = true; + entityTypesArray.push(entityTypeControl); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + public removeAll() { + const entityTypesArray = this.entityTypesVersionLoadFormGroup.get('entityTypes') as FormArray; + entityTypesArray.clear(); + this.entityTypesVersionLoadFormGroup.updateValueAndValidity(); + } + + entityTypeText(entityTypeControl: AbstractControl): string { + const entityType: EntityType = entityTypeControl.get('entityType').value; + if (entityType) { + return this.translate.instant(entityTypeTranslations.get(entityType).typePlural); + } else { + return 'Undefined'; + } + } + + allowedEntityTypes(entityTypeControl?: AbstractControl): Array { + let res = [...exportableEntityTypes]; + const currentEntityType: EntityType = entityTypeControl?.get('entityType')?.value; + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + const usedEntityTypes = value.map(val => val.entityType).filter(val => val); + res = res.filter(entityType => !usedEntityTypes.includes(entityType) || entityType === currentEntityType); + return res; + } + + onRemoveOtherEntities(removeOtherEntitiesCheckbox: MatCheckbox, entityTypeControl: AbstractControl, $event: Event) { + const removeOtherEntities: boolean = entityTypeControl.get('config.removeOtherEntities').value; + if (!removeOtherEntities) { + $event.preventDefault(); + $event.stopPropagation(); + const trigger = $('.mat-checkbox-frame', removeOtherEntitiesCheckbox._elementRef.nativeElement)[0]; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const removeOtherEntitiesConfirmPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, RemoveOtherEntitiesConfirmComponent, 'bottom', true, null, + { + onClose: (result: boolean | null) => { + removeOtherEntitiesConfirmPopover.hide(); + if (result) { + entityTypeControl.get('config').get('removeOtherEntities').patchValue(true, {emitEvent: true}); + } + } + }, {}, {}, {}, false); + } + } + } + + private updateModel() { + const value: [{entityType: string, config: EntityTypeVersionLoadConfig}] = + this.entityTypesVersionLoadFormGroup.get('entityTypes').value || []; + let modelValue: {[entityType: string]: EntityTypeVersionLoadConfig} = null; + if (value && value.length) { + modelValue = {}; + value.forEach((val) => { + modelValue[val.entityType] = val.config; + }); + } + this.modelValue = modelValue; + this.propagateChange(this.modelValue); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html new file mode 100644 index 0000000000..e9c6122a46 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.html @@ -0,0 +1,91 @@ + +
+
+ +

{{ 'version-control.create-entity-version' | translate }}

+ + + + +
+
+
+ + + + version-control.version-name + + + {{ 'version-control.version-name-required' | translate }} + + + + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
+
+
+
+ + +
+
+
+
+
+
{{ resultMessage }}
+
+ +
+
+
+ +
+
+ version-control.creating-version + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts new file mode 100644 index 0000000000..6706d80c71 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -0,0 +1,147 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnDestroy, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + SingleEntityVersionCreateRequest, + VersionCreateRequestType, + VersionCreationResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { Observable, of, Subscription } from 'rxjs'; +import { EntityType } from '@shared/models/entity-type.models'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { share } from 'rxjs/operators'; +import { parseHttpErrorMessage } from '@core/utils'; + +@Component({ + selector: 'tb-entity-version-create', + templateUrl: './entity-version-create.component.html', + styleUrls: ['./version-control.scss'] +}) +export class EntityVersionCreateComponent extends PageComponent implements OnInit, OnDestroy { + + @Input() + branch: string; + + @Input() + entityId: EntityId; + + @Input() + entityName: string; + + @Input() + onClose: (result: VersionCreationResult | null, branch: string | null) => void; + + @Input() + onBeforeCreateVersion: () => Observable; + + @Input() + popoverComponent: TbPopoverComponent; + + createVersionFormGroup: FormGroup; + + entityTypes = EntityType; + + resultMessage: string; + + versionCreateResult$: Observable; + + private versionCreateResultSubscription: Subscription; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private translate: TranslateService, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.createVersionFormGroup = this.fb.group({ + branch: [this.branch, [Validators.required]], + versionName: [this.translate.instant('version-control.default-create-entity-version-name', + {entityName: this.entityName}), [Validators.required]], + saveRelations: [false, []], + saveAttributes: [true, []], + saveCredentials: [true, []] + }); + } + + ngOnDestroy() { + super.ngOnDestroy(); + if (this.versionCreateResultSubscription) { + this.versionCreateResultSubscription.unsubscribe(); + } + } + + cancel(): void { + if (this.onClose) { + this.onClose(null, null); + } + } + + export(): void { + const before = this.onBeforeCreateVersion ? this.onBeforeCreateVersion() : of(null); + before.subscribe(() => { + const request: SingleEntityVersionCreateRequest = { + entityId: this.entityId, + branch: this.createVersionFormGroup.get('branch').value, + versionName: this.createVersionFormGroup.get('versionName').value, + config: { + saveRelations: this.createVersionFormGroup.get('saveRelations').value, + saveAttributes: this.createVersionFormGroup.get('saveAttributes').value, + saveCredentials: this.entityId.entityType === EntityType.DEVICE ? this.createVersionFormGroup.get('saveCredentials').value : false + }, + type: VersionCreateRequestType.SINGLE_ENTITY + }; + this.versionCreateResult$ = this.entitiesVersionControlService.saveEntitiesVersion(request, {ignoreErrors: true}).pipe( + share() + ); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + + this.versionCreateResultSubscription = this.versionCreateResult$.subscribe((result) => { + if (result.done) { + if (!result.added && !result.modified || result.error) { + this.resultMessage = result.error ? result.error : this.translate.instant('version-control.nothing-to-commit'); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else if (this.onClose) { + this.onClose(result, request.branch); + } + } + }, + (error) => { + this.resultMessage = parseHttpErrorMessage(error, this.translate).message; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html new file mode 100644 index 0000000000..22bebd716f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.html @@ -0,0 +1,71 @@ + +
+ +

{{ 'version-control.diff-entity-with-version' | translate: {versionName} }}

+ + + + + {{ 'version-control.differences' | translate : {count: diffCount} }} + + + + +
+
{{ 'version-control.current' | translate }}
+
+
{{ versionIdContent() }}
+
+ +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss new file mode 100644 index 0000000000..cf0bf939b7 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss @@ -0,0 +1,65 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import '../../../../../scss/constants'; + +.entity-version-diff-view { + position: relative; + &:not(.tb-fullscreen) { + &.content-ready { + width: 600px; + @media #{$mat-gt-sm} { + width: 800px; + } + @media #{$mat-gt-md} { + width: 1000px + } + @media #{$mat-gt-xmd} { + width: 1200px + } + } + } + &.tb-fullscreen { + height: 100%; + background: #fff; + .bottom-panel { + margin-bottom: 16px; + } + } + .mat-toolbar { + background: #fff; + .mat-divider-vertical { + height: 40px; + margin-right: 16px; + margin-left: 16px; + } + .diff-count { + font-size: 16px; + } + } + .version-title { + margin-left: 4px; + } + .diff-viewer { + position: relative; + border: 1px solid #c0c0c0; + margin-top: 4px; + margin-bottom: 16px; + .acediff__diffLine { + z-index: 1; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts new file mode 100644 index 0000000000..29955af18e --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -0,0 +1,328 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + ChangeDetectorRef, + Component, + ElementRef, + EventEmitter, + Input, + OnDestroy, + OnInit, + Output, + Renderer2, + ViewChild, + ViewContainerRef, + ViewEncapsulation +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { getAceDiff } from '@shared/models/ace/ace.models'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { entityExportDataToJsonString, VersionLoadResult } from '@shared/models/vc.models'; +import { Ace } from 'ace-builds'; +import { MatButton } from '@angular/material/button'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; + +interface DiffInfo { + leftStartLine: number; + leftEndLine: number; + rightStartLine: number; + rightEndLine: number; +} + +@Component({ + selector: 'tb-entity-version-diff', + templateUrl: './entity-version-diff.component.html', + styleUrls: ['./entity-version-diff.component.scss'], + encapsulation: ViewEncapsulation.None +}) +export class EntityVersionDiffComponent extends PageComponent implements OnInit, OnDestroy { + + @ViewChild('diffViewer', {static: true}) + diffViewerElmRef: ElementRef; + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + entityId: EntityId; + + @Input() + externalEntityId: EntityId; + + @Output() + versionRestored = new EventEmitter(); + + @Input() + onClose: () => void; + + @Input() + popoverComponent: TbPopoverComponent; + + differ: AceDiff; + + contentReady = false; + + preferredDiffHeight = '332px'; + + isFullscreen = false; + + hasNext = false; + hasPrevious = false; + + diffCount = 0; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private renderer: Renderer2, + private elementRef: ElementRef, + private viewContainerRef: ViewContainerRef, + private popoverService: TbPopoverService) { + super(store); + } + + ngOnInit(): void { + this.entitiesVersionControlService + .compareEntityDataToVersion(this.branch, this.entityId, this.versionId).subscribe((diffData) => { + const leftContent = entityExportDataToJsonString(diffData.currentVersion); + const rightContent = entityExportDataToJsonString(diffData.otherVersion); + const leftLines = leftContent.split('\n').length; + const rightLines = leftContent.split('\n').length; + const totalLines = Math.max(leftLines, rightLines); + let preferredLines = Math.max(10, totalLines); + preferredLines = Math.min(40, preferredLines); + this.preferredDiffHeight = (132 + preferredLines * 16) + 'px'; + getAceDiff().subscribe((aceDiff) => { + this.contentReady = true; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + setTimeout(() => { + this.differ = new aceDiff.default( + { + element: this.diffViewerElmRef.nativeElement, + mode: 'ace/mode/json', + left: { + copyLinkEnabled: false, + editable: false, + content: leftContent + }, + right: { + copyLinkEnabled: false, + editable: false, + content: rightContent + } + } as AceDiff.AceDiffConstructorOpts + ); + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + leftEditor.setShowFoldWidgets(false); + rightEditor.setShowFoldWidgets(false); + $('.acediff__left .ace_scrollbar-v', this.elementRef.nativeElement).on('scroll', () => { + rightEditor.getSession().setScrollTop(leftEditor.getSession().getScrollTop()); + }); + $('.acediff__right .ace_scrollbar-v', this.elementRef.nativeElement).on('scroll', () => { + leftEditor.getSession().setScrollTop(rightEditor.getSession().getScrollTop()); + }); + $('.acediff__left .ace_scrollbar-h', this.elementRef.nativeElement).on('scroll', () => { + rightEditor.getSession().setScrollLeft(leftEditor.getSession().getScrollLeft()); + }); + $('.acediff__right .ace_scrollbar-h', this.elementRef.nativeElement).on('scroll', () => { + leftEditor.getSession().setScrollLeft(rightEditor.getSession().getScrollLeft()); + }); + leftEditor.getSession().getSelection().on('changeCursor', () => this.updateHasNextAndPrevious()); + rightEditor.getSession().getSelection().on('changeCursor', () => this.updateHasNextAndPrevious()); + setTimeout(() => { + this.diffCount = this.differ.getNumDiffs(); + this.updateHasNextAndPrevious(); + }); + }); + }); + }); + } + + versionIdContent(): string { + let versionId = this.versionId; + if (versionId.length > 7) { + versionId = versionId.slice(0, 7); + } + return versionId + ' (' + this.versionName + ')'; + } + + prevDifference($event: Event) { + if ($event) { + $event.stopPropagation(); + } + this.moveToDiff(false); + } + + nextDifference($event: Event) { + if ($event) { + $event.stopPropagation(); + } + this.moveToDiff(true); + } + + private moveToDiff(next: boolean) { + const currentRow = this.getCurrentRow(); + const diff = next ? this.findNextLine(currentRow) : this.findPrevLine(currentRow); + if (diff) { + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + leftEditor.scrollToLine(diff.leftStartLine + 1, true, true, () => {}); + leftEditor.gotoLine(diff.leftStartLine + 1, 0, true); + rightEditor.scrollToLine(diff.rightStartLine + 1, true, true, () => {}); + rightEditor.gotoLine(diff.rightStartLine + 1, 0, true); + } + } + + onFullscreenChanged(fullscreen: boolean) { + if (fullscreen) { + this.resizeEditors(); + } else { + setTimeout(() => { + this.resizeEditors(); + }); + } + } + + private getDiffs(): DiffInfo[] { + if (this.differ) { + // @ts-ignore + return this.differ.diffs as DiffInfo[] || []; + } else { + return []; + } + } + + private getCurrentRow(): {row: number, left: boolean} { + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + let currentRow = 0; + let left = true; + const leftRow = leftEditor.getSession().getSelection().getCursor().row; + const rightRow = rightEditor.getSession().getSelection().getCursor().row; + if (leftRow >= leftEditor.getFirstVisibleRow() && leftRow <= leftEditor.getLastVisibleRow()) { + currentRow = leftRow; + } else if (rightRow >= rightEditor.getFirstVisibleRow() && rightRow <= rightEditor.getLastVisibleRow()) { + currentRow = rightRow; + left = false; + } else { + currentRow = leftRow; + } + return {row: currentRow, left}; + } + + private nextDiff(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + const diffs = this.getDiffs(); + return diffs.find((diff) => (currentLine.left ? diff.leftStartLine : diff.rightStartLine) > currentLine.row); + } + + private prevDiff(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + const diffs = this.getDiffs(); + return [...diffs].reverse().find((diff) => (currentLine.left ? diff.leftEndLine : diff.rightEndLine) < currentLine.row); + } + + private findNextLine(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + let res = this.nextDiff(currentLine); + const diffs = this.getDiffs(); + if (!res && diffs.length) { + res = diffs[diffs.length - 1]; + } + return res; + } + + private findPrevLine(currentLine: {row: number, left: boolean}): DiffInfo | undefined { + let res = this.prevDiff(currentLine); + const diffs = this.getDiffs(); + if (!res && diffs.length) { + res = diffs[0]; + } + return res; + } + + private updateHasNextAndPrevious() { + const currentRow = this.getCurrentRow(); + this.hasNext = !!this.nextDiff(currentRow); + this.hasPrevious = !!this.prevDiff(currentRow); + this.cd.markForCheck(); + } + + private resizeEditors() { + if (this.differ) { + this.differ.diff(); + const leftEditor: Ace.Editor = this.differ.getEditors().left; + const rightEditor: Ace.Editor = this.differ.getEditors().right; + leftEditor.resize(); + leftEditor.renderer.updateFull(); + rightEditor.resize(); + rightEditor.renderer.updateFull(); + } + } + + ngOnDestroy(): void { + if (this.differ) { + this.differ.destroy(); + this.differ = null; + } + } + + close(): void { + if (this.popoverComponent) { + this.popoverComponent.hide(); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: this.versionName, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + onClose: (result: VersionLoadResult | null) => { + restoreVersionPopover.hide(); + if (result && !result.error && result.result.length) { + this.close(); + this.versionRestored.emit(); + } + } + }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html new file mode 100644 index 0000000000..b096c1907f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -0,0 +1,81 @@ + +
+
+ +

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

+ + + + + +
+
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} + +
+
+
+
+ + +
+
+
+
+
+
+
+ +
+
+
+ +
+
+ version-control.restoring-entity-version + +
+
+ +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts new file mode 100644 index 0000000000..43d2452ea3 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.ts @@ -0,0 +1,151 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnDestroy, OnInit, Sanitizer } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { + EntityDataInfo, + SingleEntityVersionLoadRequest, VersionCreationResult, + VersionLoadRequestType, + VersionLoadResult +} from '@shared/models/vc.models'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { EntityId } from '@shared/models/id/entity-id'; +import { TranslateService } from '@ngx-translate/core'; +import { TbPopoverComponent } from '@shared/components/popover.component'; +import { delay, share } from 'rxjs/operators'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; +import { Observable, Subscription } from 'rxjs'; +import { parseHttpErrorMessage } from '@core/utils'; + +@Component({ + selector: 'tb-entity-version-restore', + templateUrl: './entity-version-restore.component.html', + styleUrls: ['./version-control.scss'] +}) +export class EntityVersionRestoreComponent extends PageComponent implements OnInit, OnDestroy { + + @Input() + branch: string; + + @Input() + versionName: string; + + @Input() + versionId: string; + + @Input() + externalEntityId: EntityId; + + @Input() + onClose: (result: VersionLoadResult | null) => void; + + @Input() + popoverComponent: TbPopoverComponent; + + entityDataInfo: EntityDataInfo = null; + + restoreFormGroup: FormGroup; + + errorMessage: SafeHtml; + + versionLoadResult$: Observable; + + private versionLoadResultSubscription: Subscription; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private cd: ChangeDetectorRef, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + } + + ngOnInit(): void { + this.restoreFormGroup = this.fb.group({ + loadAttributes: [true, []], + loadRelations: [true, []], + loadCredentials: [true, []] + }); + this.entitiesVersionControlService.getEntityDataInfo(this.externalEntityId, this.versionId).subscribe((data) => { + this.entityDataInfo = data; + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } + + ngOnDestroy() { + super.ngOnDestroy(); + if (this.versionLoadResultSubscription) { + this.versionLoadResultSubscription.unsubscribe(); + } + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + restore(): void { + const request: SingleEntityVersionLoadRequest = { + branch: this.branch, + versionId: this.versionId, + externalEntityId: this.externalEntityId, + config: { + loadRelations: this.entityDataInfo.hasRelations ? this.restoreFormGroup.get('loadRelations').value : false, + loadAttributes: this.entityDataInfo.hasAttributes ? this.restoreFormGroup.get('loadAttributes').value : false, + loadCredentials: this.entityDataInfo.hasCredentials ? this.restoreFormGroup.get('loadCredentials').value : false + }, + type: VersionLoadRequestType.SINGLE_ENTITY + }; + this.versionLoadResult$ = this.entitiesVersionControlService.loadEntitiesVersion(request, {ignoreErrors: true}).pipe( + share() + ); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + this.versionLoadResultSubscription = this.versionLoadResult$.subscribe((result) => { + if (result.done) { + if (result.error) { + this.errorMessage = this.entitiesVersionControlService.entityLoadErrorToMessage(result.error); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + } else { + if (this.onClose) { + this.onClose(result); + } + } + } + }, + (error) => { + this.errorMessage = this.sanitizer.bypassSecurityTrustHtml(parseHttpErrorMessage(error, this.translate).message); + this.cd.detectChanges(); + if (this.popoverComponent) { + this.popoverComponent.updatePosition(); + } + }); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html new file mode 100644 index 0000000000..b6accc40dc --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -0,0 +1,172 @@ + +
+
+ +
+
+ {{(singleEntityMode ? 'version-control.entity-versions' : 'version-control.versions') | translate}} + + +
+ + + + + +
+ + +
+ + +   + + + +
+ +
+ + + {{ 'version-control.created-time' | translate }} + + {{ entityVersion.timestamp | date:'yyyy-MM-dd HH:mm:ss' }} + + + + {{ 'version-control.version-id' | translate }} + + + + + + + + {{ 'version-control.version-name' | translate }} + + {{ entityVersion.name }} + + + + {{ 'version-control.author' | translate }} + + {{ entityVersion.author }} + + + + + + +
+ + + +
+ + + + +
+ {{ + singleEntityMode + ? 'version-control.no-entity-versions-text' + : 'version-control.no-versions-text' + }} + {{ 'common.loading' | translate }} +
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss new file mode 100644 index 0000000000..3256c3c956 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.scss @@ -0,0 +1,107 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import '../../../../../scss/constants'; + +:host { + width: 100%; + height: 100%; + display: block; + .tb-entity-table { + + &.tb-popover-mode { + position: relative; + width: 800px; + height: 600px; + } + + .tb-entity-table-content { + width: 100%; + height: 100%; + background: #fff; + + .mat-toolbar-tools{ + min-height: auto; + } + + .title-container{ + overflow: hidden; + } + + .tb-entity-table-title { + padding-right: 20px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .table-container { + overflow: auto; + } + + .tb-entity-table-info{ + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .button-widget-action{ + margin-left: auto; + overflow: hidden; + text-overflow: ellipsis; + } + } + } + + @media #{$mat-xs} { + .mat-toolbar { + height: auto; + min-height: 100px; + + .tb-entity-table-title{ + padding-bottom: 5px; + width: 100%; + } + } + } +} + +:host ::ng-deep { + .mat-sort-header-sorted .mat-sort-header-arrow { + opacity: 1 !important; + } + tb-branch-autocomplete { + mat-form-field { + font-size: 16px; + width: 250px; + + .mat-form-field-wrapper { + padding-bottom: 0; + } + + .mat-form-field-underline { + bottom: 0; + } + + @media #{$mat-xs} { + width: 100%; + + .mat-form-field-infix { + width: auto !important; + } + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts new file mode 100644 index 0000000000..d776939aaf --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.ts @@ -0,0 +1,445 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + ChangeDetectorRef, + Component, + ElementRef, EventEmitter, + Input, + OnDestroy, + OnInit, Output, Renderer2, + ViewChild, ViewContainerRef +} from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { EntityId, entityIdEquals } from '@shared/models/id/entity-id'; +import { CollectionViewer, DataSource } from '@angular/cdk/collections'; +import { BehaviorSubject, fromEvent, merge, Observable, of, ReplaySubject } from 'rxjs'; +import { emptyPageData, PageData } from '@shared/models/page/page-data'; +import { PageLink } from '@shared/models/page/page-link'; +import { catchError, debounceTime, distinctUntilChanged, map, tap } from 'rxjs/operators'; +import { EntityVersion, VersionCreationResult, VersionLoadResult } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatSort } from '@angular/material/sort'; +import { ResizeObserver } from '@juggle/resize-observer'; +import { hidePageSizePixelValue } from '@shared/models/constants'; +import { Direction, SortOrder } from '@shared/models/page/sort-order'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; +import { isNotEmptyStr } from '@core/utils'; +import { TbPopoverService } from '@shared/components/popover.service'; +import { EntityVersionCreateComponent } from '@home/components/vc/entity-version-create.component'; +import { MatButton } from '@angular/material/button'; +import { EntityVersionRestoreComponent } from '@home/components/vc/entity-version-restore.component'; +import { EntityVersionDiffComponent } from '@home/components/vc/entity-version-diff.component'; +import { ComplexVersionCreateComponent } from '@home/components/vc/complex-version-create.component'; +import { ComplexVersionLoadComponent } from '@home/components/vc/complex-version-load.component'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-entity-versions-table', + templateUrl: './entity-versions-table.component.html', + styleUrls: ['./entity-versions-table.component.scss'] +}) +export class EntityVersionsTableComponent extends PageComponent implements OnInit, AfterViewInit, OnDestroy { + + @ViewChild('branchAutocompleteComponent') branchAutocompleteComponent: BranchAutocompleteComponent; + + @Input() + singleEntityMode = false; + + @Input() + popoverComponent: TbPopoverComponent; + + @Input() + onBeforeCreateVersion: () => Observable; + + displayedColumns = ['timestamp', 'id', 'name', 'author', 'actions']; + pageLink: PageLink; + textSearchMode = false; + dataSource: EntityVersionsDatasource; + hidePageSize = false; + + branch: string = null; + + activeValue = false; + dirtyValue = false; + externalEntityIdValue: EntityId; + + viewsInited = false; + + private componentResize$: ResizeObserver; + + @Input() + set active(active: boolean) { + if (this.activeValue !== active) { + this.activeValue = active; + if (this.activeValue && this.dirtyValue) { + this.dirtyValue = false; + if (this.viewsInited) { + this.initFromDefaultBranch(); + } + } + } + } + + @Input() + set externalEntityId(externalEntityId: EntityId) { + if (!entityIdEquals(this.externalEntityIdValue, externalEntityId)) { + this.externalEntityIdValue = externalEntityId; + this.resetSortAndFilter(this.activeValue); + if (!this.activeValue) { + this.dirtyValue = true; + } + } + } + + @Input() + entityId: EntityId; + + @Input() + entityName: string; + + @Output() + versionRestored = new EventEmitter(); + + @ViewChild('searchInput') searchInputField: ElementRef; + + @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; + + constructor(protected store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private popoverService: TbPopoverService, + private renderer: Renderer2, + private cd: ChangeDetectorRef, + private viewContainerRef: ViewContainerRef, + private elementRef: ElementRef) { + super(store); + this.dirtyValue = !this.activeValue; + const sortOrder: SortOrder = { property: 'timestamp', direction: Direction.DESC }; + this.pageLink = new PageLink(10, 0, null, sortOrder); + this.dataSource = new EntityVersionsDatasource(this.entitiesVersionControlService); + } + + ngOnInit() { + this.componentResize$ = new ResizeObserver(() => { + const showHidePageSize = this.elementRef.nativeElement.offsetWidth < hidePageSizePixelValue; + if (showHidePageSize !== this.hidePageSize) { + this.hidePageSize = showHidePageSize; + this.cd.markForCheck(); + } + }); + this.componentResize$.observe(this.elementRef.nativeElement); + } + + ngOnDestroy() { + if (this.componentResize$) { + this.componentResize$.disconnect(); + } + } + + branchChanged(newBranch: string) { + if (isNotEmptyStr(newBranch) && this.branch !== newBranch) { + this.branch = newBranch; + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } + } + + ngAfterViewInit() { + fromEvent(this.searchInputField.nativeElement, 'keyup') + .pipe( + debounceTime(400), + distinctUntilChanged(), + tap(() => { + this.paginator.pageIndex = 0; + this.updateData(); + }) + ) + .subscribe(); + + this.sort.sortChange.subscribe(() => this.paginator.pageIndex = 0); + merge(this.sort.sortChange, this.paginator.page) + .pipe( + tap(() => this.updateData()) + ) + .subscribe(); + this.viewsInited = true; + if (!this.singleEntityMode || (this.activeValue && this.externalEntityIdValue)) { + this.initFromDefaultBranch(); + } + } + + toggleCreateVersion($event: Event, createVersionButton: MatButton) { + if ($event) { + $event.stopPropagation(); + } + const trigger = createVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const createVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionCreateComponent, 'leftTop', true, null, + { + branch: this.branch, + entityId: this.entityId, + entityName: this.entityName, + onBeforeCreateVersion: this.onBeforeCreateVersion, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + createVersionPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + createVersionPopover.tbComponentRef.instance.popoverComponent = createVersionPopover; + } + } + + toggleComplexCreateVersion($event: Event, complexCreateVersionButton: MatButton) { + if ($event) { + $event.stopPropagation(); + } + const trigger = complexCreateVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const complexCreateVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionCreateComponent, 'leftTop', true, null, + { + branch: this.branch, + onClose: (result: VersionCreationResult | null, branch: string | null) => { + complexCreateVersionPopover.hide(); + if (result) { + if (this.branch !== branch) { + this.branchChanged(branch); + } else { + this.updateData(); + } + } + } + }, {}, {}, {}, false); + complexCreateVersionPopover.tbComponentRef.instance.popoverComponent = complexCreateVersionPopover; + } + } + + toggleShowVersionDiff($event: Event, diffVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = diffVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const diffVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionDiffComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + entityId: this.entityId, + externalEntityId: this.externalEntityIdValue + }, {}, {}, {}, false); + diffVersionPopover.tbComponentRef.instance.popoverComponent = diffVersionPopover; + diffVersionPopover.tbComponentRef.instance.versionRestored.subscribe(() => { + this.versionRestored.emit(); + }); + } + } + + toggleRestoreEntityVersion($event: Event, restoreVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, EntityVersionRestoreComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + externalEntityId: this.externalEntityIdValue, + onClose: (result: VersionLoadResult | null) => { + restoreVersionPopover.hide(); + if (result && !result.error && result.result.length) { + this.versionRestored.emit(); + } + } + }, {}, {}, {}, false); + restoreVersionPopover.tbComponentRef.instance.popoverComponent = restoreVersionPopover; + } + } + + toggleRestoreEntitiesVersion($event: Event, restoreEntitiesVersionButton: MatButton, entityVersion: EntityVersion) { + if ($event) { + $event.stopPropagation(); + } + const trigger = restoreEntitiesVersionButton._elementRef.nativeElement; + if (this.popoverService.hasPopover(trigger)) { + this.popoverService.hidePopover(trigger); + } else { + const restoreEntitiesVersionPopover = this.popoverService.displayPopover(trigger, this.renderer, + this.viewContainerRef, ComplexVersionLoadComponent, 'leftTop', true, null, + { + branch: this.branch, + versionName: entityVersion.name, + versionId: entityVersion.id, + onClose: (result: VersionLoadResult | null) => { + restoreEntitiesVersionPopover.hide(); + } + }, {}, {}, {}, false); + restoreEntitiesVersionPopover.tbComponentRef.instance.popoverComponent = restoreEntitiesVersionPopover; + } + } + + versionIdContent(entityVersion: EntityVersion): string { + let versionId = entityVersion.id; + if (versionId.length > 7) { + versionId = versionId.slice(0, 7); + } + return versionId; + } + + enterFilterMode() { + this.textSearchMode = true; + this.pageLink.textSearch = ''; + setTimeout(() => { + this.searchInputField.nativeElement.focus(); + this.searchInputField.nativeElement.setSelectionRange(0, 0); + }, 10); + } + + exitFilterMode() { + this.textSearchMode = false; + this.pageLink.textSearch = null; + this.paginator.pageIndex = 0; + this.updateData(); + } + + private initFromDefaultBranch() { + if (this.branchAutocompleteComponent.isDefaultBranchSelected()) { + this.paginator.pageIndex = 0; + if (this.activeValue) { + this.updateData(); + } + } else { + this.branchAutocompleteComponent.selectDefaultBranchIfNeeded(true); + } + } + + updateData() { + this.pageLink.page = this.paginator.pageIndex; + this.pageLink.pageSize = this.paginator.pageSize; + this.pageLink.sortOrder.property = this.sort.active; + this.pageLink.sortOrder.direction = Direction[this.sort.direction.toUpperCase()]; + this.dataSource.loadEntityVersions(this.singleEntityMode, this.branch, this.externalEntityIdValue, this.pageLink); + } + + private resetSortAndFilter(update: boolean) { + this.textSearchMode = false; + this.pageLink.textSearch = null; + if (this.viewsInited) { + this.paginator.pageIndex = 0; + const sortable = this.sort.sortables.get('timestamp'); + this.sort.active = sortable.id; + this.sort.direction = 'desc'; + if (update) { + this.initFromDefaultBranch(); + } + } + } +} + +class EntityVersionsDatasource implements DataSource { + + private entityVersionsSubject = new BehaviorSubject([]); + private pageDataSubject = new BehaviorSubject>(emptyPageData()); + + public pageData$ = this.pageDataSubject.asObservable(); + + public dataLoading = true; + + constructor(private entitiesVersionControlService: EntitiesVersionControlService) {} + + connect(collectionViewer: CollectionViewer): Observable> { + return this.entityVersionsSubject.asObservable(); + } + + disconnect(collectionViewer: CollectionViewer): void { + this.entityVersionsSubject.complete(); + this.pageDataSubject.complete(); + } + + loadEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + this.dataLoading = true; + const result = new ReplaySubject>(); + this.fetchEntityVersions(singleEntityMode, branch, externalEntityId, pageLink).pipe( + catchError(() => of(emptyPageData())), + ).subscribe( + (pageData) => { + this.entityVersionsSubject.next(pageData.data); + this.pageDataSubject.next(pageData); + result.next(pageData); + this.dataLoading = false; + } + ); + return result; + } + + fetchEntityVersions(singleEntityMode: boolean, + branch: string, externalEntityId: EntityId, + pageLink: PageLink): Observable> { + if (!branch) { + return of(emptyPageData()); + } else { + if (singleEntityMode) { + if (externalEntityId) { + return this.entitiesVersionControlService.listEntityVersions(pageLink, branch, externalEntityId, {ignoreErrors: true}); + } else { + return of(emptyPageData()); + } + } else { + return this.entitiesVersionControlService.listVersions(pageLink, branch, {ignoreErrors: true}); + } + } + } + + isEmpty(): Observable { + return this.entityVersionsSubject.pipe( + map((entityVersions) => !entityVersions.length) + ); + } + + total(): Observable { + return this.pageDataSubject.pipe( + map((pageData) => pageData.totalElements) + ); + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html new file mode 100644 index 0000000000..14ae91e2f2 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.html @@ -0,0 +1,41 @@ + +
+
+
+
+ + + +
+
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts new file mode 100644 index 0000000000..f569d96fac --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/remove-other-entities-confirm.component.ts @@ -0,0 +1,66 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { TranslateService } from '@ngx-translate/core'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; + +@Component({ + selector: 'tb-remove-other-entities-confirm', + templateUrl: './remove-other-entities-confirm.component.html', + styleUrls: [] +}) +export class RemoveOtherEntitiesConfirmComponent extends PageComponent implements OnInit { + + @Input() + onClose: (result: boolean | null) => void; + + confirmFormGroup: FormGroup; + + removeOtherEntitiesConfirmText: SafeHtml; + + removeOtherEntitiesVerificationText = 'remove other entities'; + + constructor(protected store: Store, + private translate: TranslateService, + private sanitizer: DomSanitizer, + private fb: FormBuilder) { + super(store); + this.removeOtherEntitiesConfirmText = this.sanitizer.bypassSecurityTrustHtml(this.translate.instant('version-control.remove-other-entities-confirm-text')); + } + + ngOnInit(): void { + this.confirmFormGroup = this.fb.group({ + verification: [null, []] + }); + } + + cancel(): void { + if (this.onClose) { + this.onClose(null); + } + } + + confirm(): void { + if (this.onClose) { + this.onClose(true); + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html new file mode 100644 index 0000000000..3e8bd271f3 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.html @@ -0,0 +1,110 @@ + +
+ + +
+ admin.repository-settings + +
+
+ + + +
+ +
+
+ + admin.repository-url + + + admin.repository-url-required + + + + admin.default-branch + + +
+ admin.authentication-settings + + admin.auth-method + + + {{repositoryAuthMethodTranslations.get(method) | translate}} + + + +
+ + common.username + + + + {{ 'admin.change-password-access-token' | translate }} + + + admin.password-access-token + + + +
+
+ + + + {{ 'admin.change-passphrase' | translate }} + + + admin.passphrase + + + +
+
+
+ + + + +
+
+
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss new file mode 100644 index 0000000000..77ae0ffd8f --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.scss @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-card.repository-settings { + margin: 8px; + } + .fields-group { + padding: 0 16px 8px; + margin-bottom: 10px; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + width: fit-content; + } + + legend + * { + display: block; + margin-top: 16px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts new file mode 100644 index 0000000000..1120185db9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/repository-settings.component.ts @@ -0,0 +1,221 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { ChangeDetectorRef, Component, Input, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { FormBuilder, FormGroup, FormGroupDirective, Validators } from '@angular/forms'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AdminService } from '@core/http/admin.service'; +import { + RepositorySettings, + RepositoryAuthMethod, + repositoryAuthMethodTranslationMap +} from '@shared/models/settings.models'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { TranslateService } from '@ngx-translate/core'; +import { isNotEmptyStr } from '@core/utils'; +import { DialogService } from '@core/services/dialog.service'; +import { ActionAuthUpdateHasRepository } from '@core/auth/auth.actions'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { catchError, mergeMap, take } from 'rxjs/operators'; +import { of } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-repository-settings', + templateUrl: './repository-settings.component.html', + styleUrls: ['./repository-settings.component.scss', './../../pages/admin/settings-card.scss'] +}) +export class RepositorySettingsComponent extends PageComponent implements OnInit { + + @Input() + detailsMode = false; + + @Input() + popoverComponent: TbPopoverComponent; + + repositorySettingsForm: FormGroup; + settings: RepositorySettings = null; + + repositoryAuthMethod = RepositoryAuthMethod; + repositoryAuthMethods = Object.values(RepositoryAuthMethod); + repositoryAuthMethodTranslations = repositoryAuthMethodTranslationMap; + + showChangePassword = false; + changePassword = false; + + showChangePrivateKeyPassword = false; + changePrivateKeyPassword = false; + + constructor(protected store: Store, + private adminService: AdminService, + private dialogService: DialogService, + private translate: TranslateService, + private cd: ChangeDetectorRef, + public fb: FormBuilder) { + super(store); + } + + ngOnInit() { + this.repositorySettingsForm = this.fb.group({ + repositoryUri: [null, [Validators.required]], + defaultBranch: ['main', []], + authMethod: [RepositoryAuthMethod.USERNAME_PASSWORD, [Validators.required]], + username: [null, []], + password: [null, []], + privateKeyFileName: [null, [Validators.required]], + privateKey: [null, []], + privateKeyPassword: [null, []] + }); + this.updateValidators(false); + this.repositorySettingsForm.get('authMethod').valueChanges.subscribe(() => { + this.updateValidators(true); + }); + this.repositorySettingsForm.get('privateKeyFileName').valueChanges.subscribe(() => { + this.updateValidators(false); + }); + this.store.pipe( + select(selectHasRepository), + take(1), + mergeMap((hasRepository) => { + if (hasRepository) { + return this.adminService.getRepositorySettings({ignoreErrors: true}).pipe( + catchError(() => of(null)) + ); + } else { + return of(null); + } + }) + ).subscribe( + (settings) => { + this.settings = settings; + if (this.settings != null) { + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + } else { + this.showChangePrivateKeyPassword = true; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + } + }); + } + + checkAccess(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.checkRepositoryAccess(settings).subscribe(() => { + this.store.dispatch(new ActionNotificationShow({ message: this.translate.instant('admin.check-repository-access-success'), + type: 'success' })); + }); + } + + save(): void { + const settings: RepositorySettings = this.repositorySettingsForm.value; + this.adminService.saveRepositorySettings(settings).subscribe( + (savedSettings) => { + this.settings = savedSettings; + if (this.settings.authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.showChangePassword = true; + this.changePassword = false; + } else { + this.showChangePrivateKeyPassword = true; + this.changePrivateKeyPassword = false; + } + this.repositorySettingsForm.reset(this.settings); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: true })); + } + ); + } + + delete(formDirective: FormGroupDirective): void { + this.dialogService.confirm( + this.translate.instant('admin.delete-repository-settings-title', ), + this.translate.instant('admin.delete-repository-settings-text'), null, + this.translate.instant('action.delete') + ).subscribe((data) => { + if (data) { + this.adminService.deleteRepositorySettings().subscribe( + () => { + this.settings = null; + this.showChangePassword = false; + this.changePassword = false; + this.showChangePrivateKeyPassword = false; + this.changePrivateKeyPassword = false; + formDirective.resetForm(); + this.repositorySettingsForm.reset({ defaultBranch: 'main', authMethod: RepositoryAuthMethod.USERNAME_PASSWORD }); + this.updateValidators(false); + this.store.dispatch(new ActionAuthUpdateHasRepository({ hasRepository: false })); + } + ); + } + }); + } + + changePasswordChanged() { + if (this.changePassword) { + this.repositorySettingsForm.get('password').patchValue(''); + this.repositorySettingsForm.get('password').markAsDirty(); + } + this.updateValidators(false); + } + + changePrivateKeyPasswordChanged() { + if (this.changePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').patchValue(''); + this.repositorySettingsForm.get('privateKeyPassword').markAsDirty(); + } + this.updateValidators(false); + } + + updateValidators(emitEvent?: boolean): void { + const authMethod: RepositoryAuthMethod = this.repositorySettingsForm.get('authMethod').value; + const privateKeyFileName: string = this.repositorySettingsForm.get('privateKeyFileName').value; + if (authMethod === RepositoryAuthMethod.USERNAME_PASSWORD) { + this.repositorySettingsForm.get('username').enable({emitEvent}); + if (this.changePassword || !this.showChangePassword) { + this.repositorySettingsForm.get('password').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('password').disable({emitEvent}); + } + this.repositorySettingsForm.get('privateKeyFileName').disable({emitEvent}); + this.repositorySettingsForm.get('privateKey').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } else { + this.repositorySettingsForm.get('username').disable({emitEvent}); + this.repositorySettingsForm.get('password').disable({emitEvent}); + this.repositorySettingsForm.get('privateKeyFileName').enable({emitEvent}); + this.repositorySettingsForm.get('privateKey').enable({emitEvent}); + if (this.changePrivateKeyPassword || !this.showChangePrivateKeyPassword) { + this.repositorySettingsForm.get('privateKeyPassword').enable({emitEvent}); + } else { + this.repositorySettingsForm.get('privateKeyPassword').disable({emitEvent}); + } + if (isNotEmptyStr(privateKeyFileName)) { + this.repositorySettingsForm.get('privateKey').clearValidators(); + } else { + this.repositorySettingsForm.get('privateKey').setValidators([Validators.required]); + } + } + this.repositorySettingsForm.get('username').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('password').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyFileName').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKey').updateValueAndValidity({emitEvent: false}); + this.repositorySettingsForm.get('privateKeyPassword').updateValueAndValidity({emitEvent: false}); + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.html b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html new file mode 100644 index 0000000000..eaf80fd664 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.html @@ -0,0 +1,31 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss new file mode 100644 index 0000000000..da8df4b469 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.scss @@ -0,0 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts new file mode 100644 index 0000000000..59816b7b5d --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.component.ts @@ -0,0 +1,78 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; +import { FormGroup } from '@angular/forms'; +import { EntityId } from '@shared/models/id/entity-id'; +import { Observable } from 'rxjs'; +import { TbPopoverComponent } from '@shared/components/popover.component'; + +@Component({ + selector: 'tb-version-control', + templateUrl: './version-control.component.html', + styleUrls: ['./version-control.component.scss'] +}) +export class VersionControlComponent implements OnInit, HasConfirmForm { + + @ViewChild('repositorySettingsComponent', {static: false}) repositorySettingsComponent: RepositorySettingsComponent; + + @Input() + detailsMode = false; + + @Input() + popoverComponent: TbPopoverComponent; + + @Input() + active = true; + + @Input() + singleEntityMode = false; + + @Input() + externalEntityId: EntityId; + + @Input() + entityId: EntityId; + + @Input() + entityName: string; + + @Input() + onBeforeCreateVersion: () => Observable; + + @Output() + versionRestored = new EventEmitter(); + + hasRepository$ = this.store.pipe(select(selectHasRepository)); + + constructor(private store: Store) { + + } + + ngOnInit() { + + } + + confirmForm(): FormGroup { + return this.repositorySettingsComponent?.repositorySettingsForm; + } + +} diff --git a/ui-ngx/src/app/modules/home/components/vc/version-control.scss b/ui-ngx/src/app/modules/home/components/vc/version-control.scss new file mode 100644 index 0000000000..d0d96be941 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/vc/version-control.scss @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .vc-result-message { + padding: 0 8px; + text-align: center; + &:first-child { + padding-top: 48px; + } + &:nth-last-child(2) { + padding-bottom: 8px; + } + &.error { + text-align: start; + font-weight: 400; + } + &.progress { + padding-bottom: 32px; + } + } +} diff --git a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html index bf7a27d4af..6e08046a22 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.html @@ -50,6 +50,9 @@ {{ 'legend.show-total' | translate }} + + {{ 'legend.show-latest' | translate }} + {{ 'legend.sort-legend' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts index 588e1237e7..16e32bc866 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/legend-config.component.ts @@ -66,7 +66,8 @@ export class LegendConfigComponent implements OnInit, OnDestroy, ControlValueAcc showMin: [null, []], showMax: [null, []], showAvg: [null, []], - showTotal: [null, []] + showTotal: [null, []], + showLatest: [null, []] }); this.legendSettingsFormDirectionChanges$ = this.legendConfigForm.get('direction').valueChanges .subscribe((direction: LegendDirection) => { @@ -124,7 +125,8 @@ export class LegendConfigComponent implements OnInit, OnDestroy, ControlValueAcc showMin: isDefined(legendConfig.showMin) ? legendConfig.showMin : false, showMax: isDefined(legendConfig.showMax) ? legendConfig.showMax : false, showAvg: isDefined(legendConfig.showAvg) ? legendConfig.showAvg : false, - showTotal: isDefined(legendConfig.showTotal) ? legendConfig.showTotal : false + showTotal: isDefined(legendConfig.showTotal) ? legendConfig.showTotal : false, + showLatest: isDefined(legendConfig.showLatest) ? legendConfig.showLatest : false }, {emitEvent: false}); } this.onDirectionChanged(legendConfig.direction); diff --git a/ui-ngx/src/app/modules/home/components/widget/legend.component.html b/ui-ngx/src/app/modules/home/components/widget/legend.component.html index 0b02d3389c..e92c5c4baf 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/legend.component.html @@ -23,6 +23,7 @@ {{ 'legend.max' | translate }} {{ 'legend.avg' | translate }} {{ 'legend.total' | translate }} + {{ 'legend.latest' | translate }} @@ -38,6 +39,7 @@ {{ legendData.data[legendKey.dataIndex].max }} {{ legendData.data[legendKey.dataIndex].avg }} {{ legendData.data[legendKey.dataIndex].total }} + {{ legendData.data[legendKey.dataIndex].latest }} @@ -75,5 +77,11 @@ {{ legendData.data[legendKey.dataIndex].total }} + + {{ 'legend.latest' | translate }} + + {{ legendData.data[legendKey.dataIndex].latest }} + + diff --git a/ui-ngx/src/app/modules/home/components/widget/legend.component.ts b/ui-ngx/src/app/modules/home/components/widget/legend.component.ts index 0074280732..4548621517 100644 --- a/ui-ngx/src/app/modules/home/components/widget/legend.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/legend.component.ts @@ -43,7 +43,8 @@ export class LegendComponent implements OnInit { this.displayHeader = this.legendConfig.showMin === true || this.legendConfig.showMax === true || this.legendConfig.showAvg === true || - this.legendConfig.showTotal === true; + this.legendConfig.showTotal === true || + this.legendConfig.showLatest === true; this.isHorizontal = this.legendConfig.position === LegendPosition.bottom || this.legendConfig.position === LegendPosition.top; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts index 783ebd2096..dd03604a9a 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/maps/providers/image-map.ts @@ -27,7 +27,7 @@ import { WidgetContext } from '@home/models/widget-component.models'; import { DataSet, DatasourceType, widgetType } from '@shared/models/widget.models'; import { DataKeyType } from '@shared/models/telemetry/telemetry.models'; import { WidgetSubscriptionOptions } from '@core/api/widget-api.models'; -import { isDefinedAndNotNull, isEmptyStr, parseFunction } from '@core/utils'; +import { isDefinedAndNotNull, isEmptyStr, isNotEmptyStr, parseFunction } from '@core/utils'; import { EntityDataPageLink } from '@shared/models/query/query.models'; const maxZoom = 4; // ? @@ -93,10 +93,12 @@ export class ImageMap extends LeafletMap { type: widgetType.latest, callbacks: { onDataUpdated: (subscription) => { - if (subscription.data[0]?.data[0]?.length > 0) { + if (isNotEmptyStr(subscription.data[0]?.data[0]?.[1])) { result.next([subscription.data[0].data, isUpdate]); - isUpdate = true; + } else { + result.next([[[0, options.mapImageUrl]], isUpdate]); } + isUpdate = true; } } }; diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts index 5beef7ea8c..b9b294cccd 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/rpc/persistent-table.component.ts @@ -42,7 +42,7 @@ import { import cssjs from '@core/css/css'; import { UtilsService } from '@core/services/utils.service'; import { TranslateService } from '@ngx-translate/core'; -import { hashCode, isDefined, isNumber } from '@core/utils'; +import { hashCode, isDefined, isNumber, parseHttpErrorMessage } from '@core/utils'; import { CollectionViewer, DataSource } from '@angular/cdk/collections'; import { emptyPageData, PageData } from '@shared/models/page/page-data'; import { @@ -520,7 +520,7 @@ class PersistentDatasource implements DataSource { this.subscription.rpcErrorText = 'Request Timeout.'; } else { this.subscription.rpcErrorText = 'Error : ' + rejection.status + ' - ' + rejection.statusText; - const error = this.extractRejectionErrorText(rejection); + const error = parseHttpErrorMessage(rejection, this.translate); if (error) { this.subscription.rpcErrorText += '
'; this.subscription.rpcErrorText += error.message || ''; @@ -534,40 +534,6 @@ class PersistentDatasource implements DataSource { return rpcSubject.asObservable(); } - extractRejectionErrorText(rejection: HttpErrorResponse) { - let error = null; - if (rejection.error) { - error = rejection.error; - try { - error = rejection.error ? JSON.parse(rejection.error) : null; - } catch (e) {} - } - if (error && !error.message) { - error = this.prepareMessageFromData(error); - } else if (error && error.message) { - error = error.message; - } - return error; - } - - prepareMessageFromData(data) { - if (typeof data === 'object' && data.constructor === ArrayBuffer) { - const msg = String.fromCharCode.apply(null, new Uint8Array(data)); - try { - const msgObj = JSON.parse(msg); - if (msgObj.message) { - return msgObj.message; - } else { - return msg; - } - } catch (e) { - return msg; - } - } else { - return data; - } - } - isEmpty(): Observable { return this.persistentSubject.pipe( map((requests) => !requests.length) diff --git a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts index 6dc8746b2e..ff08af8039 100644 --- a/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts +++ b/ui-ngx/src/app/modules/home/dialogs/home-dialogs.service.ts @@ -53,4 +53,5 @@ export class HomeDialogsService { } }).afterClosed(); } + } diff --git a/ui-ngx/src/app/modules/home/models/widget-component.models.ts b/ui-ngx/src/app/modules/home/models/widget-component.models.ts index 465cade9f2..4e250b71ea 100644 --- a/ui-ngx/src/app/modules/home/models/widget-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/widget-component.models.ts @@ -50,7 +50,6 @@ import { WidgetTypeId } from '@shared/models/id/widget-type-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetLayout } from '@shared/models/dashboard.models'; import { formatValue, isDefined } from '@core/utils'; -import { forkJoin, of } from 'rxjs'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { @@ -79,7 +78,9 @@ import { PageLink } from '@shared/models/page/page-link'; import { SortOrder } from '@shared/models/page/sort-order'; import { DomSanitizer } from '@angular/platform-browser'; import { Router } from '@angular/router'; -import { catchError, map, mergeMap, switchMap } from 'rxjs/operators'; +import { EdgeService } from '@core/http/edge.service'; +import * as RxJS from 'rxjs'; +import * as RxJSOperators from 'rxjs/operators'; import { TbPopoverComponent } from '@shared/components/popover.component'; import { EntityId } from '@shared/models/id/entity-id'; @@ -160,6 +161,7 @@ export class WidgetContext { deviceService: DeviceService; assetService: AssetService; entityViewService: EntityViewService; + edgeService: EdgeService; customerService: CustomerService; dashboardService: DashboardService; userService: UserService; @@ -203,7 +205,7 @@ export class WidgetContext { if (this.defaultSubscription) { return this.defaultSubscription.sendOneWayCommand(method, params, timeout, persistent, retries, additionalInfo, requestUUID); } else { - return of(null); + return RxJS.of(null); } }, sendTwoWayCommand: (method, params, timeout, persistent, @@ -211,14 +213,14 @@ export class WidgetContext { if (this.defaultSubscription) { return this.defaultSubscription.sendTwoWayCommand(method, params, timeout, persistent, retries, additionalInfo, requestUUID); } else { - return of(null); + return RxJS.of(null); } }, completedCommand: () => { if (this.defaultSubscription) { return this.defaultSubscription.completedCommand(); } else { - return of(null); + return RxJS.of(null); } } }; @@ -266,12 +268,9 @@ export class WidgetContext { private popoverComponents: TbPopoverComponent[] = []; rxjs = { - forkJoin, - of, - map, - mergeMap, - switchMap, - catchError + + ...RxJS, + ...RxJSOperators }; registerPopoverComponent(popoverComponent: TbPopoverComponent) { diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index e74bb0ae4d..84020bc491 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -33,6 +33,8 @@ import { EntityDetailsPageComponent } from '@home/components/entity/entity-detai import { entityDetailsPageBreadcrumbLabelFunction } from '@home/pages/home-pages.models'; import { BreadCrumbConfig } from '@shared/components/breadcrumb'; import { QueuesTableConfigResolver } from '@home/pages/admin/queue/queues-table-config.resolver'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; +import { AutoCommitAdminSettingsComponent } from '@home/pages/admin/auto-commit-admin-settings.component'; import { TwoFactorAuthSettingsComponent } from '@home/pages/admin/two-factor-auth-settings.component'; @Injectable() @@ -237,6 +239,32 @@ const routes: Routes = [ isMdiIcon: true } } + }, + { + path: 'repository', + component: RepositoryAdminSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.repository-settings', + breadcrumb: { + label: 'admin.repository-settings', + icon: 'manage_history' + } + } + }, + { + path: 'auto-commit', + component: AutoCommitAdminSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.TENANT_ADMIN], + title: 'admin.auto-commit-settings', + breadcrumb: { + label: 'admin.auto-commit-settings', + icon: 'settings_backup_restore' + } + } } ] } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index a6e9393611..54ffbc61b2 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -29,6 +29,8 @@ import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dial import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { QueueComponent} from '@home/pages/admin/queue/queue.component'; +import { RepositoryAdminSettingsComponent } from '@home/pages/admin/repository-admin-settings.component'; +import { AutoCommitAdminSettingsComponent } from '@home/pages/admin/auto-commit-admin-settings.component'; import { TwoFactorAuthSettingsComponent } from '@home/pages/admin/two-factor-auth-settings.component'; @NgModule({ @@ -43,6 +45,8 @@ import { TwoFactorAuthSettingsComponent } from '@home/pages/admin/two-factor-aut HomeSettingsComponent, ResourcesLibraryComponent, QueueComponent, + RepositoryAdminSettingsComponent, + AutoCommitAdminSettingsComponent, TwoFactorAuthSettingsComponent ], imports: [ diff --git a/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html new file mode 100644 index 0000000000..63060f6a0a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.html @@ -0,0 +1,23 @@ + + + + + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts new file mode 100644 index 0000000000..6c5c110884 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/auto-commit-admin-settings.component.ts @@ -0,0 +1,51 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { select, Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { AutoCommitSettingsComponent } from '@home/components/vc/auto-commit-settings.component'; +import { selectHasRepository } from '@core/auth/auth.selectors'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; + +@Component({ + selector: 'tb-auto-commit-admin-settings', + templateUrl: './auto-commit-admin-settings.component.html', + styleUrls: [] +}) +export class AutoCommitAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('repositorySettingsComponent', {static: false}) repositorySettingsComponent: RepositorySettingsComponent; + @ViewChild('autoCommitSettingsComponent', {static: false}) autoCommitSettingsComponent: AutoCommitSettingsComponent; + + hasRepository$ = this.store.pipe(select(selectHasRepository)); + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.repositorySettingsComponent ? + this.repositorySettingsComponent?.repositorySettingsForm : + this.autoCommitSettingsComponent?.autoCommitSettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html new file mode 100644 index 0000000000..7b408fc93a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.html @@ -0,0 +1,18 @@ + + diff --git a/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts new file mode 100644 index 0000000000..1ee0a7288a --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/admin/repository-admin-settings.component.ts @@ -0,0 +1,44 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit, ViewChild } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { FormGroup } from '@angular/forms'; +import { RepositorySettingsComponent } from '@home/components/vc/repository-settings.component'; + +@Component({ + selector: 'tb-repository-admin-settings', + templateUrl: './repository-admin-settings.component.html', + styleUrls: [] +}) +export class RepositoryAdminSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + @ViewChild('repositorySettingsComponent') repositorySettingsComponent: RepositorySettingsComponent; + + constructor(protected store: Store) { + super(store); + } + + ngOnInit() { + } + + confirmForm(): FormGroup { + return this.repositorySettingsComponent?.repositorySettingsForm; + } +} diff --git a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html index e5accc15a9..452dce8bcd 100644 --- a/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/asset/asset-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html index 9c97f1d96d..942ea2a73b 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/customer/customer-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts index a3716823e1..bbdde763e2 100644 --- a/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/customer/customers-table-config.resolver.ts @@ -34,6 +34,7 @@ import { CustomerTabsComponent } from '@home/pages/customer/customer-tabs.compon import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; +import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; @Injectable() export class CustomersTableConfigResolver implements Resolve> { @@ -41,6 +42,7 @@ export class CustomersTableConfigResolver implements Resolve = new EntityTableConfig(); constructor(private customerService: CustomerService, + private homeDialogs: HomeDialogsService, private translate: TranslateService, private datePipe: DatePipe, private router: Router, diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html index 0e6fa55e8c..a9fc2f644c 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboard-tabs.component.html @@ -19,3 +19,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts index f51043fb3b..fddc9e94ce 100644 --- a/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/dashboard/dashboards-table-config.resolver.ts @@ -69,6 +69,7 @@ import { AddEntitiesToEdgeDialogComponent, AddEntitiesToEdgeDialogData } from '@home/dialogs/add-entities-to-edge-dialog.component'; +import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; @Injectable() export class DashboardsTableConfigResolver implements Resolve> { @@ -80,6 +81,7 @@ export class DashboardsTableConfigResolver implements Resolve + + + diff --git a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts index 2434f50151..597e2a6390 100644 --- a/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/device-profile/device-profiles-table-config.resolver.ts @@ -42,6 +42,7 @@ import { AddDeviceProfileDialogData } from '@home/components/profile/add-device-profile-dialog.component'; import { ImportExportService } from '@home/components/import-export/import-export.service'; +import { HomeDialogsService } from '@home/dialogs/home-dialogs.service'; @Injectable() export class DeviceProfilesTableConfigResolver implements Resolve> { @@ -50,6 +51,7 @@ export class DeviceProfilesTableConfigResolver implements Resolve + + + diff --git a/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html b/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html index 7ba3c80805..79307ddcb2 100644 --- a/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html +++ b/ui-ngx/src/app/modules/home/pages/entity-view/entity-view-tabs.component.html @@ -49,3 +49,9 @@ label="{{ 'audit-log.audit-logs' | translate }}" #auditLogsTab="matTab"> + + + diff --git a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts index cf3aec6191..1b45d32a63 100644 --- a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts @@ -37,6 +37,7 @@ import { DeviceProfileModule } from './device-profile/device-profile.module'; import { ApiUsageModule } from '@home/pages/api-usage/api-usage.module'; import { EdgeModule } from '@home/pages/edge/edge.module'; import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; +import { VcModule } from '@home/pages/vc/vc.module'; @NgModule({ exports: [ @@ -58,7 +59,8 @@ import { OtaUpdateModule } from '@home/pages/ota-update/ota-update.module'; AuditLogModule, ApiUsageModule, OtaUpdateModule, - UserModule + UserModule, + VcModule ], providers: [ { diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html index baea44755c..a8ae57a458 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

rulenode.add

: {{ruleNode.component.name}} diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.scss b/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.scss new file mode 100644 index 0000000000..2057868837 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/rulechain/add-rule-node-dialog.component.scss @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import './../scss/constants'; + +:host { + .dialog-container { + min-width: 650px !important; + + @media #{$mat-lt-md} { + min-width: 100% !important; + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html index 49eb3953f5..84d768c61d 100644 --- a/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html +++ b/ui-ngx/src/app/modules/home/pages/rulechain/rulechain-page.component.html @@ -164,6 +164,17 @@ + + + + check + + + {{ 'version-control.default' | translate }} + + + + {{ 'version-control.branch-required' | translate }} + + diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.scss b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.scss new file mode 100644 index 0000000000..06b464db27 --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.scss @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +.mat-option.branch-option { + .mat-icon, .check-placeholder { + margin-right: 8px; + } + .check-placeholder { + width: 18px; + display: inline-block; + } + .mat-option-text { + width: 100%; + .default-branch { + float: right; + } + } +} diff --git a/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts new file mode 100644 index 0000000000..c135b0a8f7 --- /dev/null +++ b/ui-ngx/src/app/shared/components/vc/branch-autocomplete.component.ts @@ -0,0 +1,296 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { + AfterViewInit, + Component, + ElementRef, + forwardRef, + Input, + NgZone, + OnInit, + ViewChild, + ViewEncapsulation +} from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { Observable } from 'rxjs'; +import { debounceTime, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; +import { Store } from '@ngrx/store'; +import { AppState } from '@app/core/core.state'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { BranchInfo } from '@shared/models/vc.models'; +import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; +import { isNotEmptyStr } from '@core/utils'; +import { MatAutocomplete, MatAutocompleteTrigger } from '@angular/material/autocomplete'; + +@Component({ + selector: 'tb-branch-autocomplete', + templateUrl: './branch-autocomplete.component.html', + styleUrls: ['./branch-autocomplete.component.scss'], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => BranchAutocompleteComponent), + multi: true + }], + encapsulation: ViewEncapsulation.None +}) +export class BranchAutocompleteComponent implements ControlValueAccessor, OnInit, AfterViewInit { + + branchFormGroup: FormGroup; + + modelValue: string | null; + + private requiredValue: boolean; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + private disabledValue: boolean; + + get disabled(): boolean { + return this.disabledValue; + } + + @Input() + set disabled(value: boolean) { + this.disabledValue = coerceBooleanProperty(value); + if (this.disabledValue) { + this.branchFormGroup.disable({emitEvent: false}); + } else { + this.branchFormGroup.enable({emitEvent: false}); + } + } + + @Input() + selectDefaultBranch = true; + + @Input() + selectionMode = false; + + @Input() + emptyPlaceholder: string; + + @ViewChild('branchAutocomplete') matAutocomplete: MatAutocomplete; + @ViewChild('branchInput', { read: MatAutocompleteTrigger, static: true }) autoCompleteTrigger: MatAutocompleteTrigger; + @ViewChild('branchInput', {static: true}) branchInput: ElementRef; + + filteredBranches: Observable>; + + defaultBranch: BranchInfo = null; + + searchText = ''; + + loading = false; + + private dirty = false; + + private clearButtonClicked = false; + + private propagateChange = (v: any) => { }; + + constructor(private store: Store, + private entitiesVersionControlService: EntitiesVersionControlService, + private fb: FormBuilder, + private zone: NgZone) { + this.branchFormGroup = this.fb.group({ + branch: [null, []] + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.filteredBranches = this.branchFormGroup.get('branch').valueChanges + .pipe( + tap((value: BranchInfo | string) => { + let modelValue: BranchInfo | null; + if (typeof value === 'string' || !value) { + if (!this.selectionMode && typeof value === 'string' && isNotEmptyStr(value)) { + modelValue = {name: value, default: false}; + } else { + modelValue = null; + } + } else { + modelValue = value; + } + if (!this.selectionMode || modelValue) { + this.updateView(modelValue); + } + }), + map(value => { + if (value) { + if (typeof value === 'string') { + return value; + } else { + return value.name; + } + } else { + return ''; + } + }), + debounceTime(150), + distinctUntilChanged(), + switchMap(name => this.fetchBranches(name)), + share() + ); + } + + ngAfterViewInit(): void { + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + } + + isDefaultBranchSelected(): boolean { + return this.defaultBranch && this.defaultBranch.name === this.modelValue; + } + + selectDefaultBranchIfNeeded(force = false): void { + if ((this.selectDefaultBranch && !this.modelValue) || force) { + setTimeout(() => { + if (this.defaultBranch) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); + } else { + this.loading = true; + this.getBranches().subscribe( + () => { + if (this.defaultBranch || force) { + this.branchFormGroup.get('branch').patchValue(this.defaultBranch, {emitEvent: false}); + this.modelValue = this.defaultBranch?.name; + this.propagateChange(this.modelValue); + this.loading = false; + } else { + this.loading = false; + } + } + ); + } + }); + } + } + + writeValue(value: string | null): void { + this.searchText = ''; + this.modelValue = value; + if (value != null) { + this.branchFormGroup.get('branch').patchValue({name: value}, {emitEvent: false}); + } else { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: false}); + this.selectDefaultBranchIfNeeded(); + } + this.dirty = true; + } + + onFocus() { + if (this.dirty) { + this.branchFormGroup.get('branch').updateValueAndValidity({onlySelf: true, emitEvent: true}); + this.dirty = false; + } + } + + onBlur() { + if (this.clearButtonClicked) { + this.clearButtonClicked = false; + } else if (!this.matAutocomplete.isOpen) { + this.selectAvailableValue(); + } + } + + onPanelClosed() { + this.selectAvailableValue(); + } + + selectAvailableValue() { + if (this.selectionMode) { + const branch = this.branchFormGroup.get('branch').value; + this.getBranches().pipe( + map(branches => { + let foundBranch = branches.find(b => b.name === branch); + if (!foundBranch && isNotEmptyStr(this.modelValue)) { + foundBranch = branches.find(b => b.name === this.modelValue); + } + return foundBranch; + }) + ).subscribe((val) => { + if (!val && this.defaultBranch) { + val = this.defaultBranch; + } + this.zone.run(() => { + this.branchFormGroup.get('branch').patchValue(val, {emitEvent: true}); + }, 0); + }); + } + } + + updateView(value: BranchInfo | null) { + if (this.modelValue !== value?.name) { + this.modelValue = value?.name; + this.propagateChange(this.modelValue); + } + } + + displayBranchFn(branch?: BranchInfo): string | undefined { + return branch ? branch.name : undefined; + } + + private fetchBranches(searchText?: string): Observable> { + this.searchText = searchText; + return this.getBranches().pipe( + map(branches => { + let res = branches.filter(branch => { + return searchText ? branch.name.toUpperCase().startsWith(searchText.toUpperCase()) : true; + }); + if (!this.selectionMode && isNotEmptyStr(searchText) && !res.find(b => b.name === searchText)) { + res = [{name: searchText, default: false}, ...res]; + } + return res; + } + ) + ); + } + + private getBranches(): Observable> { + return this.entitiesVersionControlService.listBranches().pipe( + tap((data) => { + this.defaultBranch = data.find(branch => branch.default); + }) + ); + } + + clear() { + this.clearButtonClicked = true; + setTimeout(() => { + this.branchFormGroup.get('branch').patchValue(null, {emitEvent: true}); + this.branchInput.nativeElement.blur(); + this.branchInput.nativeElement.focus(); + }, 0); + } + +} diff --git a/ui-ngx/src/app/shared/models/ace/ace.models.ts b/ui-ngx/src/app/shared/models/ace/ace.models.ts index 835c8d1cfa..9777935922 100644 --- a/ui-ngx/src/app/shared/models/ace/ace.models.ts +++ b/ui-ngx/src/app/shared/models/ace/ace.models.ts @@ -21,6 +21,7 @@ import { map, mergeMap, tap } from 'rxjs/operators'; let aceDependenciesLoaded = false; let aceModule: any; +let aceDiffModule: any; function loadAceDependencies(): Observable { if (aceDependenciesLoaded) { @@ -74,6 +75,21 @@ export function getAce(): Observable { } } +export function getAceDiff(): Observable { + if (aceDiffModule) { + return of(aceDiffModule); + } else { + return getAce().pipe( + mergeMap((ace) => { + return from(import('ace-diff')); + }), + tap((module) => { + aceDiffModule = module; + }) + ); + } +} + export class Range implements Ace.Range { public start: Ace.Point; diff --git a/ui-ngx/src/app/shared/models/asset.models.ts b/ui-ngx/src/app/shared/models/asset.models.ts index d38aee4d96..115eea4610 100644 --- a/ui-ngx/src/app/shared/models/asset.models.ts +++ b/ui-ngx/src/app/shared/models/asset.models.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { AssetId } from './id/asset-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; import { EntitySearchQuery } from '@shared/models/relation.models'; -export interface Asset extends BaseData { +export interface Asset extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/base-data.ts b/ui-ngx/src/app/shared/models/base-data.ts index 3af9bb9059..7b3ea9a642 100644 --- a/ui-ngx/src/app/shared/models/base-data.ts +++ b/ui-ngx/src/app/shared/models/base-data.ts @@ -27,6 +27,12 @@ export interface BaseData { label?: string; } +export interface ExportableEntity { + createdTime?: number; + id?: T; + externalId?: T; +} + export function hasIdEquals(id1: HasId, id2: HasId): boolean { if (isDefinedAndNotNull(id1) && isDefinedAndNotNull(id2)) { return id1.id === id2.id; diff --git a/ui-ngx/src/app/shared/models/constants.ts b/ui-ngx/src/app/shared/models/constants.ts index 37027edc68..28c08f09a0 100644 --- a/ui-ngx/src/app/shared/models/constants.ts +++ b/ui-ngx/src/app/shared/models/constants.ts @@ -16,6 +16,7 @@ import { InjectionToken } from '@angular/core'; import { IModulesMap } from '@modules/common/modules-map.models'; +import { EntityType } from '@shared/models/entity-type.models'; export const Constants = { serverErrorCode: { @@ -38,6 +39,20 @@ export const Constants = { } }; +export const serverErrorCodesTranslations = new Map([ + [Constants.serverErrorCode.general, 'server-error.general'], + [Constants.serverErrorCode.authentication, 'server-error.authentication'], + [Constants.serverErrorCode.jwtTokenExpired, 'server-error.jwt-token-expired'], + [Constants.serverErrorCode.tenantTrialExpired, 'server-error.tenant-trial-expired'], + [Constants.serverErrorCode.credentialsExpired, 'server-error.credentials-expired'], + [Constants.serverErrorCode.permissionDenied, 'server-error.permission-denied'], + [Constants.serverErrorCode.invalidArguments, 'server-error.invalid-arguments'], + [Constants.serverErrorCode.badRequestParams, 'server-error.bad-request-params'], + [Constants.serverErrorCode.itemNotFound, 'server-error.item-not-found'], + [Constants.serverErrorCode.tooManyRequests, 'server-error.too-many-requests'], + [Constants.serverErrorCode.tooManyUpdates, 'server-error.too-many-updates'], +]); + export const MediaBreakpoints = { xs: 'screen and (max-width: 599px)', sm: 'screen and (min-width: 600px) and (max-width: 959px)', @@ -135,7 +150,9 @@ export const HelpLinks = { widgetsConfigStatic: helpBaseUrl + '/docs/user-guide/ui/dashboards#static', ruleNodePushToCloud: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-cloud', ruleNodePushToEdge: helpBaseUrl + '/docs/user-guide/rule-engine-2-0/action-nodes/#push-to-edge', - queue: helpBaseUrl + '/docs/user-guide/queue' + queue: helpBaseUrl + '/docs/user-guide/queue', + repositorySettings: helpBaseUrl + '/docs/user-guide/ui/repository-settings', + autoCommitSettings: helpBaseUrl + '/docs/user-guide/ui/auto-commit-settings', } }; diff --git a/ui-ngx/src/app/shared/models/customer.model.ts b/ui-ngx/src/app/shared/models/customer.model.ts index 435edf263f..67c1429189 100644 --- a/ui-ngx/src/app/shared/models/customer.model.ts +++ b/ui-ngx/src/app/shared/models/customer.model.ts @@ -17,8 +17,9 @@ import { CustomerId } from '@shared/models/id/customer-id'; import { ContactBased } from '@shared/models/contact-based.model'; import { TenantId } from './id/tenant-id'; +import { ExportableEntity } from '@shared/models/base-data'; -export interface Customer extends ContactBased { +export interface Customer extends ContactBased, ExportableEntity { tenantId: TenantId; title: string; additionalInfo?: any; diff --git a/ui-ngx/src/app/shared/models/dashboard.models.ts b/ui-ngx/src/app/shared/models/dashboard.models.ts index 4332e484f6..12e1c83ada 100644 --- a/ui-ngx/src/app/shared/models/dashboard.models.ts +++ b/ui-ngx/src/app/shared/models/dashboard.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DashboardId } from '@shared/models/id/dashboard-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { ShortCustomerInfo } from '@shared/models/customer.model'; @@ -23,7 +23,7 @@ import { Timewindow } from '@shared/models/time/time.models'; import { EntityAliases } from './alias.models'; import { Filters } from '@shared/models/query/query.models'; -export interface DashboardInfo extends BaseData { +export interface DashboardInfo extends BaseData, ExportableEntity { tenantId?: TenantId; title?: string; image?: string; diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index 1d6ffa13af..88a2ccf16e 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { DeviceId } from './id/device-id'; import { TenantId } from '@shared/models/id/tenant-id'; import { CustomerId } from '@shared/models/id/customer-id'; @@ -564,7 +564,7 @@ export interface DeviceProfileData { provisionConfiguration?: DeviceProvisionConfiguration; } -export interface DeviceProfile extends BaseData { +export interface DeviceProfile extends BaseData, ExportableEntity { tenantId?: TenantId; name: string; description?: string; @@ -689,7 +689,7 @@ export interface DeviceData { transportConfiguration: DeviceTransportConfiguration; } -export interface Device extends BaseData { +export interface Device extends BaseData, ExportableEntity { tenantId?: TenantId; customerId?: CustomerId; name: string; diff --git a/ui-ngx/src/app/shared/models/entity-type.models.ts b/ui-ngx/src/app/shared/models/entity-type.models.ts index c92c2c43db..65753f3962 100644 --- a/ui-ngx/src/app/shared/models/entity-type.models.ts +++ b/ui-ngx/src/app/shared/models/entity-type.models.ts @@ -251,6 +251,9 @@ export const entityTypeTranslations = new Map { +export interface EntityView extends BaseData, ExportableEntity { tenantId: TenantId; customerId: CustomerId; entityId: EntityId; diff --git a/ui-ngx/src/app/shared/models/rule-chain.models.ts b/ui-ngx/src/app/shared/models/rule-chain.models.ts index 85d6047191..01f9d268c3 100644 --- a/ui-ngx/src/app/shared/models/rule-chain.models.ts +++ b/ui-ngx/src/app/shared/models/rule-chain.models.ts @@ -14,14 +14,14 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { RuleChainId } from '@shared/models/id/rule-chain-id'; import { RuleNodeId } from '@shared/models/id/rule-node-id'; import { RuleNode, RuleNodeComponentDescriptor, RuleNodeType } from '@shared/models/rule-node.models'; import { ComponentType } from '@shared/models/component-descriptor.models'; -export interface RuleChain extends BaseData { +export interface RuleChain extends BaseData, ExportableEntity { tenantId: TenantId; name: string; firstRuleNodeId: RuleNodeId; diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index e00c8e28a0..c1a7cae87a 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -16,6 +16,7 @@ import { ValidatorFn } from '@angular/forms'; import { isNotEmptyStr, isNumber } from '@core/utils'; +import { VersionCreateConfig } from '@shared/models/vc.models'; export const smtpPortPattern: RegExp = /^([0-9]{1,4}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])$/; @@ -396,3 +397,30 @@ export function createSmsProviderConfiguration(type: SmsProviderType): SmsProvid } return smsProviderConfiguration; } + +export enum RepositoryAuthMethod { + USERNAME_PASSWORD = 'USERNAME_PASSWORD', + PRIVATE_KEY = 'PRIVATE_KEY' +} + +export const repositoryAuthMethodTranslationMap = new Map([ + [RepositoryAuthMethod.USERNAME_PASSWORD, 'admin.auth-method-username-password'], + [RepositoryAuthMethod.PRIVATE_KEY, 'admin.auth-method-private-key'] +]); + +export interface RepositorySettings { + repositoryUri: string; + defaultBranch: string; + authMethod: RepositoryAuthMethod; + username: string; + password: string; + privateKeyFileName: string; + privateKey: string; + privateKeyPassword: string; +} + +export interface AutoVersionCreateConfig extends VersionCreateConfig { + branch: string; +} + +export type AutoCommitSettings = {[entityType: string]: AutoVersionCreateConfig}; diff --git a/ui-ngx/src/app/shared/models/tenant.model.ts b/ui-ngx/src/app/shared/models/tenant.model.ts index 5f91707a26..70bbb1dc28 100644 --- a/ui-ngx/src/app/shared/models/tenant.model.ts +++ b/ui-ngx/src/app/shared/models/tenant.model.ts @@ -41,6 +41,9 @@ export interface DefaultTenantProfileConfiguration { transportDeviceTelemetryMsgRateLimit?: string; transportDeviceTelemetryDataPointsRateLimit?: string; + tenantEntityExportRateLimit?: string; + tenantEntityImportRateLimit?: string; + maxTransportMessages: number; maxTransportDataPoints: number; maxREExecutions: number; diff --git a/ui-ngx/src/app/shared/models/vc.models.ts b/ui-ngx/src/app/shared/models/vc.models.ts new file mode 100644 index 0000000000..98f66f74a9 --- /dev/null +++ b/ui-ngx/src/app/shared/models/vc.models.ts @@ -0,0 +1,242 @@ +/// +/// Copyright © 2016-2022 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { EntityId } from '@shared/models/id/entity-id'; +import { EntityType } from '@shared/models/entity-type.models'; +import { ExportableEntity } from '@shared/models/base-data'; +import { EntityRelation } from '@shared/models/relation.models'; +import { Device, DeviceCredentials } from '@shared/models/device.models'; +import { RuleChain, RuleChainMetaData } from '@shared/models/rule-chain.models'; + +export const exportableEntityTypes: Array = [ + EntityType.ASSET, + EntityType.DEVICE, + EntityType.ENTITY_VIEW, + EntityType.DASHBOARD, + EntityType.CUSTOMER, + EntityType.DEVICE_PROFILE, + EntityType.RULE_CHAIN, + EntityType.WIDGETS_BUNDLE +]; + +export interface VersionCreateConfig { + saveRelations: boolean; + saveAttributes: boolean; + saveCredentials: boolean; +} + +export enum VersionCreateRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + COMPLEX = 'COMPLEX' +} + +export interface VersionCreateRequest { + versionName: string; + branch: string; + type: VersionCreateRequestType; +} + +export interface SingleEntityVersionCreateRequest extends VersionCreateRequest { + entityId: EntityId; + config: VersionCreateConfig; + type: VersionCreateRequestType.SINGLE_ENTITY; +} + +export enum SyncStrategy { + MERGE = 'MERGE', + OVERWRITE = 'OVERWRITE' +} + +export const syncStrategyTranslationMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite'] + ] +); + +export const syncStrategyHintMap = new Map( + [ + [SyncStrategy.MERGE, 'version-control.sync-strategy-merge-hint'], + [SyncStrategy.OVERWRITE, 'version-control.sync-strategy-overwrite-hint'] + ] +); + +export interface EntityTypeVersionCreateConfig extends VersionCreateConfig { + syncStrategy: SyncStrategy; + entityIds: string[]; + allEntities: boolean; +} + +export interface ComplexVersionCreateRequest extends VersionCreateRequest { + syncStrategy: SyncStrategy; + entityTypes: {[entityType: string]: EntityTypeVersionCreateConfig}; + type: VersionCreateRequestType.COMPLEX; +} + +export function createDefaultEntityTypesVersionCreate(): {[entityType: string]: EntityTypeVersionCreateConfig} { + const res: {[entityType: string]: EntityTypeVersionCreateConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + syncStrategy: null, + saveAttributes: true, + saveRelations: true, + saveCredentials: true, + allEntities: true, + entityIds: [] + }; + } + return res; +} + +export interface VersionLoadConfig { + loadRelations: boolean; + loadAttributes: boolean; + loadCredentials: boolean; +} + +export enum VersionLoadRequestType { + SINGLE_ENTITY = 'SINGLE_ENTITY', + ENTITY_TYPE = 'ENTITY_TYPE' +} + +export interface VersionLoadRequest { + branch: string; + versionId: string; + type: VersionLoadRequestType; +} + +export interface SingleEntityVersionLoadRequest extends VersionLoadRequest { + externalEntityId: EntityId; + config: VersionLoadConfig; + type: VersionLoadRequestType.SINGLE_ENTITY; +} + +export interface EntityTypeVersionLoadConfig extends VersionLoadConfig { + removeOtherEntities: boolean; + findExistingEntityByName: boolean; +} + +export interface EntityTypeVersionLoadRequest extends VersionLoadRequest { + entityTypes: {[entityType: string]: EntityTypeVersionLoadConfig}; + type: VersionLoadRequestType.ENTITY_TYPE; +} + +export function createDefaultEntityTypesVersionLoad(): {[entityType: string]: EntityTypeVersionLoadConfig} { + const res: {[entityType: string]: EntityTypeVersionLoadConfig} = {}; + for (const entityType of exportableEntityTypes) { + res[entityType] = { + loadAttributes: true, + loadRelations: true, + loadCredentials: true, + removeOtherEntities: false, + findExistingEntityByName: true + }; + } + return res; +} + +export interface BranchInfo { + name: string; + default: boolean; +} + +export interface EntityVersion { + timestamp: number; + id: string; + name: string; + author: string; +} + +export interface VersionCreationResult { + version: EntityVersion; + added: number; + modified: number; + removed: number; + error: string; + done: boolean; +} + +export interface EntityTypeLoadResult { + entityType: EntityType; + created: number; + updated: number; + deleted: number; +} + +export enum EntityLoadErrorType { + DEVICE_CREDENTIALS_CONFLICT = 'DEVICE_CREDENTIALS_CONFLICT', + MISSING_REFERENCED_ENTITY = 'MISSING_REFERENCED_ENTITY' +} + +export const entityLoadErrorTranslationMap = new Map( + [ + [EntityLoadErrorType.DEVICE_CREDENTIALS_CONFLICT, 'version-control.device-credentials-conflict'], + [EntityLoadErrorType.MISSING_REFERENCED_ENTITY, 'version-control.missing-referenced-entity'] + ] +); + +export interface EntityLoadError { + type: EntityLoadErrorType; + source: EntityId; + target: EntityId; +} + +export interface VersionLoadResult { + result: Array; + error: EntityLoadError; + done: boolean; +} + +export interface AttributeExportData { + key: string; + lastUpdateTs: number; + booleanValue: boolean; + strValue: string; + longValue: number; + doubleValue: number; + jsonValue: string; +} + +export interface EntityExportData> { + entity: E; + entityType: EntityType; + relations: Array; + attributes: {[key: string]: Array}; +} + +export interface DeviceExportData extends EntityExportData { + credentials: DeviceCredentials; +} + +export interface RuleChainExportData extends EntityExportData { + metaData: RuleChainMetaData; +} + +export interface EntityDataDiff { + currentVersion: EntityExportData; + otherVersion: EntityExportData; + rawDiff: string; +} + +export function entityExportDataToJsonString(data: EntityExportData): string { + return JSON.stringify(data, null, 4); +} + +export interface EntityDataInfo { + hasRelations: boolean; + hasAttributes: boolean; + hasCredentials: boolean; +} diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index 12167a2f18..35b3b0fc21 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -243,6 +243,7 @@ export interface LegendConfig { showMax: boolean; showAvg: boolean; showTotal: boolean; + showLatest: boolean; } export function defaultLegendConfig(wType: widgetType): LegendConfig { @@ -253,7 +254,8 @@ export function defaultLegendConfig(wType: widgetType): LegendConfig { showMin: false, showMax: false, showAvg: wType === widgetType.timeseries, - showTotal: false + showTotal: false, + showLatest: false }; } @@ -362,6 +364,7 @@ export interface LegendKeyData { max: string; avg: string; total: string; + latest: string; hidden: boolean; } diff --git a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts index 106438a591..235190ead2 100644 --- a/ui-ngx/src/app/shared/models/widgets-bundle.model.ts +++ b/ui-ngx/src/app/shared/models/widgets-bundle.model.ts @@ -14,11 +14,11 @@ /// limitations under the License. /// -import { BaseData } from '@shared/models/base-data'; +import { BaseData, ExportableEntity } from '@shared/models/base-data'; import { TenantId } from '@shared/models/id/tenant-id'; import { WidgetsBundleId } from '@shared/models/id/widgets-bundle-id'; -export interface WidgetsBundle extends BaseData { +export interface WidgetsBundle extends BaseData, ExportableEntity { tenantId: TenantId; alias: string; title: string; diff --git a/ui-ngx/src/app/shared/shared.module.ts b/ui-ngx/src/app/shared/shared.module.ts index 81f2ff0afb..aaa29f02c0 100644 --- a/ui-ngx/src/app/shared/shared.module.ts +++ b/ui-ngx/src/app/shared/shared.module.ts @@ -163,6 +163,7 @@ import { HtmlComponent } from '@shared/components/html.component'; import { SafePipe } from '@shared/pipe/safe.pipe'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { MultipleImageInputComponent } from '@shared/components/multiple-image-input.component'; +import { BranchAutocompleteComponent } from '@shared/components/vc/branch-autocomplete.component'; import { PhoneInputComponent } from '@shared/components/phone-input.component'; export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) { @@ -286,6 +287,7 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) CopyButtonComponent, TogglePasswordComponent, ProtobufContentComponent, + BranchAutocompleteComponent, PhoneInputComponent ], imports: [ @@ -487,6 +489,7 @@ export function MarkedOptionsFactory(markedOptionsService: MarkedOptionsService) CopyButtonComponent, TogglePasswordComponent, ProtobufContentComponent, + BranchAutocompleteComponent, PhoneInputComponent ] }) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cfc919721d..68fcf8b186 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -59,7 +59,9 @@ "read-more": "Read more", "hide": "Hide", "done": "Done", - "print": "Print" + "print": "Print", + "restore": "Restore", + "confirm": "Confirm" }, "aggregation": { "aggregation": "Aggregation", @@ -322,6 +324,30 @@ "queue-submit-strategy": "Submit strategy", "queue-processing-strategy": "Processing strategy", "queue-configuration": "Queue configuration", + "repository-settings": "Repository settings", + "repository-url": "Repository URL", + "repository-url-required": "Repository URL is required.", + "default-branch": "Default branch name", + "authentication-settings": "Authentication settings", + "auth-method": "Authentication method", + "auth-method-username-password": "Password / access token", + "auth-method-private-key": "Private key", + "password-access-token": "Password / access token", + "change-password-access-token": "Change password / access token", + "private-key": "Private key", + "drop-private-key-file-or": "Drag and drop a private key file or", + "passphrase": "Passphrase", + "enter-passphrase": "Enter passphrase", + "change-passphrase": "Change passphrase", + "check-access": "Check access", + "check-repository-access-success": "Repository access successfully verified!", + "delete-repository-settings-title": "Are you sure you want to delete repository settings?", + "delete-repository-settings-text": "Be careful, after the confirmation the repository settings will be removed and version control feature will be unavailable.", + "auto-commit-settings": "Auto-commit settings", + "auto-commit-entities": "Auto-commit entities", + "no-auto-commit-entities-prompt": "No entities configured for auto-commit", + "delete-auto-commit-settings-title": "Are you sure you want to delete auto-commit settings?", + "delete-auto-commit-settings-text": "Be careful, after the confirmation the auto-commit settings will be removed and auto-commit will be disabled for all entities.", "2fa": { "2fa": "Two-factor authentication", "available-providers": "Available providers", @@ -1819,6 +1845,9 @@ "type-current-tenant": "Current Tenant", "type-current-user": "Current User", "type-current-user-owner": "Current User Owner", + "type-widgets-bundle": "Widgets bundle", + "type-widgets-bundles": "Widgets bundles", + "list-of-widgets-bundles": "{ count, plural, 1 {One widgets bundle} other {List of # widget bundles} }", "search": "Search entities", "selected-entities": "{ count, plural, 1 {1 entity} other {# entities} } selected", "entity-name": "Entity name", @@ -2472,11 +2501,13 @@ "show-min": "Show min value", "show-avg": "Show average value", "show-total": "Show total value", + "show-latest": "Show latest value", "settings": "Legend settings", "min": "min", "max": "max", "avg": "avg", "total": "total", + "latest": "latest", "comparison-time-ago": { "previousInterval": "(previous interval)", "customInterval": "(custom interval)", @@ -2928,18 +2959,25 @@ "add" : "Add queue", "details": "Queue details", "topic": "Topic", - "submit-strategy": "Submit Strategy", - "processing-strategy": "Processing Strategy", + "submit-settings": "Submit settings", + "submit-strategy": "Strategy type *", + "grouping-parameter": "Grouping parameter", + "processing-settings": "Retries processing settings", + "processing-strategy": "Processing type *", + "retries-settings": "Retries settings", + "polling-settings": "Polling settings", + "batch-processing": "Batch processing", "poll-interval": "Poll interval", "partitions": "Partitions", - "consumer-per-partition": "Consumer per partition", + "immediate-processing": "Immediate processing", + "consumer-per-partition": "Send message poll for each consumer", "consumer-per-partition-hint": "Enable separate consumer(s) per each partition", - "processing-timeout": "Processing timeout, ms", + "processing-timeout": "Processing within, ms", "batch-size": "Batch size", - "retries": "Retries (0 - unlimited)", - "failure-percentage": "Failure Percentage", - "pause-between-retries": "Pause between retries", - "max-pause-between-retries": "Maximal pause between retries", + "retries": "Number of retries (0 – unlimited)", + "failure-percentage": "Percentage of failure messages for skipping retries", + "pause-between-retries": "Retry within, sec", + "max-pause-between-retries": "Additional retry within, sec", "delete": "Delete queue", "copyId": "Copy queue Id", "idCopiedMessage": "Queue Id has been copied to clipboard", @@ -2971,6 +3009,19 @@ "retry-failed-and-timeout-hint": "Retry all failed and timed-out messages from processing pack" } }, + "server-error": { + "general": "General server error", + "authentication": "Authentication error", + "jwt-token-expired": "JWT token expired", + "tenant-trial-expired": "Tenant trial expired", + "credentials-expired": "Credentials expired", + "permission-denied": "Permission denied", + "invalid-arguments": "Invalid arguments", + "bad-request-params": "Bad request params", + "item-not-found": "Item not found", + "too-many-requests": "Too many requests", + "too-many-updates": "Too many updates" + }, "tenant": { "tenant": "Tenant", "tenants": "Tenants", @@ -3041,90 +3092,101 @@ "export-failed-error": "Unable to export tenant profile: {{error}}", "tenant-profile-file": "Tenant profile file", "invalid-tenant-profile-file-error": "Unable to import tenant profile: Invalid tenant profile data structure.", - "maximum-devices": "Maximum number of devices (0 - unlimited)", - "maximum-devices-required": "Maximum number of devices is required.", - "maximum-devices-range": "Minimum number of devices can't be negative", - "maximum-assets": "Maximum number of assets (0 - unlimited)", - "maximum-assets-required": "Maximum number of assets is required.", - "maximum-assets-range": "Maximum number of assets can't be negative", - "maximum-customers": "Maximum number of customers (0 - unlimited)", - "maximum-customers-required": "Maximum number of customers is required.", - "maximum-customers-range": "Maximum number of customers can't be negative", - "maximum-users": "Maximum number of users (0 - unlimited)", - "maximum-users-required": "Maximum number of users is required.", - "maximum-users-range": "Maximum number of users can't be negative", - "maximum-dashboards": "Maximum number of dashboards (0 - unlimited)", - "maximum-dashboards-required": "Maximum number of dashboards is required.", - "maximum-dashboards-range": "Maximum number of dashboards can't be negative", - "maximum-rule-chains": "Maximum number of rule chains (0 - unlimited)", - "maximum-rule-chains-required": "Maximum number of rule chains is required.", - "maximum-rule-chains-range": "Maximum number of rule chains can't be negative", - "maximum-resources-sum-data-size": "Maximum sum of resource files size in bytes (0 - unlimited)", - "maximum-resources-sum-data-size-required": "Maximum sum of resource files size is required.", - "maximum-resources-sum-data-size-range": "Maximum sum of resource files size can`t be negative", - "maximum-ota-packages-sum-data-size": "Maximum sum of ota package files size in bytes (0 - unlimited)", - "maximum-ota-package-sum-data-size-required": "Maximum sum of ota package files size is required.", - "maximum-ota-package-sum-data-size-range": "Maximum sum of ota package files size can`t be negative", - "transport-tenant-msg-rate-limit": "Transport tenant messages rate limit.", - "transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages rate limit.", - "transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points rate limit.", - "transport-device-msg-rate-limit": "Transport device messages rate limit.", - "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages rate limit.", - "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points rate limit.", - "max-transport-messages": "Maximum number of transport messages (0 - unlimited)", - "max-transport-messages-required": "Maximum number of transport messages is required.", - "max-transport-messages-range": "Maximum number of transport messages can't be negative", - "max-transport-data-points": "Maximum number of transport data points (0 - unlimited)", - "max-transport-data-points-required": "Maximum number of transport data points is required.", - "max-transport-data-points-range": "Maximum number of transport data points can't be negative", - "max-r-e-executions": "Maximum number of Rule Engine executions (0 - unlimited)", - "max-r-e-executions-required": "Maximum number of Rule Engine executions is required.", - "max-r-e-executions-range": "Maximum number of Rule Engine executions can't be negative", - "max-j-s-executions": "Maximum number of JavaScript executions (0 - unlimited)", - "max-j-s-executions-required": "Maximum number of JavaScript executions is required.", - "max-j-s-executions-range": "Maximum number of JavaScript executions can't be negative", - "max-d-p-storage-days": "Maximum number of data points storage days (0 - unlimited)", - "max-d-p-storage-days-required": "Maximum number of data points storage days is required.", - "max-d-p-storage-days-range": "Maximum number of data points storage days can't be negative", - "default-storage-ttl-days": "Default storage TTL days (0 - unlimited)", - "default-storage-ttl-days-required": "Default storage TTL days is required.", - "default-storage-ttl-days-range": "Default storage TTL days can't be negative", - "alarms-ttl-days": "Alarms TTL days (0 - unlimited)", + "advanced-settings": "Advanced settings", + "entities": "Entities", + "rule-engine": "Rule Engine", + "time-to-live": "Time-to-live", + "alarms-and-notifications": "Alarms and notifications", + "ota-files-in-bytes": "OTA files in bytes", + "ws-title": "WS", + "unlimited": "(0 - unlimited)", + "maximum-devices": "Devices maximum number", + "maximum-devices-required": "Devices maximum number is required.", + "maximum-devices-range": "Devices maximum number can't be negative", + "maximum-assets": "Assets maximum number", + "maximum-assets-required": "Assets maximum number is required.", + "maximum-assets-range": "Assets maximum number can't be negative", + "maximum-customers": "Customers maximum number", + "maximum-customers-required": "Customers maximum number is required.", + "maximum-customers-range": "Customers maximum number can't be negative", + "maximum-users": "Users maximum number", + "maximum-users-required": "Users maximum number is required.", + "maximum-users-range": "Users maximum number can't be negative", + "maximum-dashboards": "Dashboards maximum number", + "maximum-dashboards-required": "Dashboards maximum number is required.", + "maximum-dashboards-range": "Dashboards maximum number can't be negative", + "maximum-rule-chains": "Rule chains maximum number", + "maximum-rule-chains-required": "Rule chains maximum number is required.", + "maximum-rule-chains-range": "Rule chains maximum number can't be negative", + "maximum-resources-sum-data-size": "Resource files sum size", + "maximum-resources-sum-data-size-required": "Resource files sum size is required.", + "maximum-resources-sum-data-size-range": "Resource files sum size can`t be negative", + "maximum-ota-packages-sum-data-size": "OTA package files sum size", + "maximum-ota-package-sum-data-size-required": "OTA package files sum size is required.", + "maximum-ota-package-sum-data-size-range": "OTA package files sum size can`t be negative", + "transport-tenant-msg-rate-limit": "Transport tenant messages", + "transport-tenant-telemetry-msg-rate-limit": "Transport tenant telemetry messages", + "transport-tenant-telemetry-data-points-rate-limit": "Transport tenant telemetry data points", + "transport-device-msg-rate-limit": "Transport device messages", + "transport-device-telemetry-msg-rate-limit": "Transport device telemetry messages", + "transport-device-telemetry-data-points-rate-limit": "Transport device telemetry data points", + "tenant-entity-export-rate-limit": "Entity version creation", + "tenant-entity-import-rate-limit": "Entity version load", + "max-transport-messages": "Transport messages maximum number", + "max-transport-messages-required": "Transport messages maximum number is required.", + "max-transport-messages-range": "Transport messages maximum number can't be negative", + "max-transport-data-points": "Transport data points maximum number ", + "max-transport-data-points-required": "Transport data points maximum number is required.", + "max-transport-data-points-range": "Transport data points maximum number can't be negative", + "max-r-e-executions": "Rule Engine executions maximum number", + "max-r-e-executions-required": "Rule Engine executions maximum number is required.", + "max-r-e-executions-range": "Rule Engine executions maximum number can't be negative", + "max-j-s-executions": "JavaScript executions maximum number ", + "max-j-s-executions-required": "JavaScript executions maximum number is required.", + "max-j-s-executions-range": "JavaScript executions maximum number can't be negative", + "max-d-p-storage-days": "Data points storage days maximum number", + "max-d-p-storage-days-required": "Data points storage days maximum number is required.", + "max-d-p-storage-days-range": "Data points storage days maximum number can't be negative", + "default-storage-ttl-days": "Storage TTL days by default", + "default-storage-ttl-days-required": "Storage TTL days by default is required.", + "default-storage-ttl-days-range": "Storage TTL days by default can't be negative", + "alarms-ttl-days": "Alarms TTL days", "alarms-ttl-days-required": "Alarms TTL days required", "alarms-ttl-days-days-range": "Alarms TTL days can't be negative", - "rpc-ttl-days": "RPC TTL days (0 - unlimited)", + "rpc-ttl-days": "RPC TTL days", "rpc-ttl-days-required": "RPC TTL days required", "rpc-ttl-days-days-range": "RPC TTL days can't be negative", - "max-rule-node-executions-per-message": "Maximum number of rule node executions per message (0 - unlimited)", - "max-rule-node-executions-per-message-required": "Maximum number of rule node executions per message is required.", - "max-rule-node-executions-per-message-range": "Maximum number of rule node executions per message can't be negative", - "max-emails": "Maximum number of emails sent (0 - unlimited)", - "max-emails-required": "Maximum number of emails sent is required.", - "max-emails-range": "Maximum number of emails sent can't be negative", - "max-sms": "Maximum number of SMS sent (0 - unlimited)", - "max-sms-required": "Maximum number of SMS sent is required.", - "max-sms-range": "Maximum number of SMS sent can't be negative", - "max-created-alarms": "Maximum number of alarms created (0 - unlimited)", - "max-created-alarms-required": "Maximum number of alarms created is required.", - "max-created-alarms-range": "Maximum number of alarms created can't be negative", + "max-rule-node-executions-per-message": "Rule node per message executions maximum number", + "max-rule-node-executions-per-message-required": "MRule node per message executions maximum number is required.", + "max-rule-node-executions-per-message-range": "Rule node per message executions maximum number can't be negative", + "max-emails": "Emails sent maximum number", + "max-emails-required": "Emails sent maximum number is required.", + "max-emails-range": "Emails sent maximum number can't be negative", + "max-sms": "SMS sent maximum number", + "max-sms-required": "SMS sent maximum number is required.", + "max-sms-range": "SMS sent maximum number can't be negative", + "max-created-alarms": "Alarms created maximum number", + "max-created-alarms-required": "Alarms created maximum number is required.", + "max-created-alarms-range": "Alarms created maximum number be negative", "no-queue": "No Queue configured", "add-queue": "Add Queue", "queues-with-count": "Queues ({{count}})", - "tenant-rest-limits": "Rate limit for REST requests for tenant", - "customer-rest-limits": "Rate limit for REST requests for customer", + "tenant-rest-limits": "REST requests for tenant", + "customer-rest-limits": "REST requests for customer", "incorrect-pattern-for-rate-limits": "The format is comma separated pairs of capacity and period (in seconds) with a colon between, e.g. 100:1,2000:60", "too-small-value-zero": "The value must be bigger than 0", "too-small-value-one": "The value must be bigger than 1", - "cassandra-tenant-limits-configuration": "Cassandra query rate limit for tenant", - "ws-limit-max-sessions-per-tenant": "Maximum number of WS sessions per tenant", - "ws-limit-max-sessions-per-customer": "Maximum number of WS sessions per customer", - "ws-limit-max-sessions-per-public-user": "Maximum number of WS sessions per public user", - "ws-limit-queue-per-session": "Maximum size of WS message queue per session", - "ws-limit-max-subscriptions-per-tenant": "Maximum number of WS subscriptions per tenant", - "ws-limit-max-subscriptions-per-customer": "Maximum number of WS subscriptions per customer", - "ws-limit-max-subscriptions-per-regular-user": "Maximum number of WS subscriptions per regular user", - "ws-limit-max-subscriptions-per-public-user": "Maximum number of WS subscriptions per public user", - "ws-limit-updates-per-session": "Rate limit for WS updates per session" + "cassandra-tenant-limits-configuration": "Cassandra query for tenant", + "ws-limit-max-sessions-per-tenant": "Sessions per tenant maximum number", + "ws-limit-max-sessions-per-customer": "Sessions per customer maximum number", + "ws-limit-max-sessions-per-regular-user": "Sessions per regular user maximum number", + "ws-limit-max-sessions-per-public-user": "Sessions per public user maximum number", + "ws-limit-queue-per-session": "Message queue per session maximum size", + "ws-limit-max-subscriptions-per-tenant": "Subscriptions per tenant maximum number", + "ws-limit-max-subscriptions-per-customer": "Subscriptions per customer maximum number", + "ws-limit-max-subscriptions-per-regular-user": "Subscriptions per regular user maximum number", + "ws-limit-max-subscriptions-per-public-user": "Subscriptions per public user maximum number", + "ws-limit-updates-per-session": "WS updates per session" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 second} other {# seconds} }", @@ -3253,6 +3315,71 @@ "json-value-invalid": "JSON value has an invalid format", "json-value-required": "JSON value is required." }, + "version-control": { + "version-control": "Version control", + "management": "Version control management", + "branch": "Branch", + "default": "Default", + "select-branch": "Select branch", + "branch-required": "Branch is required", + "create-entity-version": "Create entity version", + "version-name": "Version name", + "version-name-required": "Version name is required", + "author": "Author", + "export-relations": "Export relations", + "export-attributes": "Export attributes", + "export-credentials": "Export credentials", + "entity-versions": "Entity versions", + "versions": "Versions", + "created-time": "Created time", + "version-id": "Version ID", + "no-entity-versions-text": "No entity versions found", + "no-versions-text": "No versions found", + "copy-full-version-id": "Copy full version id", + "create-version": "Create version", + "creating-version": "Creating version... Please wait", + "nothing-to-commit": "No changes to commit", + "restore-version": "Restore version", + "restore-entity-from-version": "Restore entity from version '{{versionName}}'", + "restoring-entity-version": "Restoring entity version... Please wait", + "load-relations": "Load relations", + "load-attributes": "Load attributes", + "load-credentials": "Load credentials", + "show-version-diff": "Show version diff", + "diff-entity-with-version": "Diff with entity version '{{versionName}}'", + "previous-difference": "Previous Difference", + "next-difference": "Next Difference", + "current": "Current", + "differences": "{ count, plural, 1 {1 difference} other {# differences} }", + "create-entities-version": "Create entities version", + "default-sync-strategy": "Default sync strategy", + "sync-strategy-merge": "Merge", + "sync-strategy-overwrite": "Overwrite", + "entities-to-export": "Entities to export", + "entities-to-restore": "Entities to restore", + "sync-strategy": "Sync strategy", + "all-entities": "All entities", + "no-entities-to-export-prompt": "Please specify entities to export", + "no-entities-to-restore-prompt": "Please specify entities to restore", + "add-entity-type": "Add entity type", + "remove-all": "Remove all", + "version-create-result": "{ added, plural, 0 {No entities} 1 {1 entity} other {# entities} } added.
{ modified, plural, 0 {No entities} 1 {1 entity} other {# entities} } modified.
{ removed, plural, 0 {No entities} 1 {1 entity} other {# entities} } removed.", + "remove-other-entities": "Remove other entities", + "find-existing-entity-by-name": "Find existing entity by name", + "restore-entities-from-version": "Restore entities from version '{{versionName}}'", + "restoring-entities-from-version": "Restoring entities... Please wait", + "no-entities-restored": "No entities restored", + "created": "{{created}} created", + "updated": "{{updated}} updated", + "deleted": "{{deleted}} deleted", + "remove-other-entities-confirm-text": "Be careful! This will permanently delete all current entities
not present in the version you want to restore.

Please type remove other entities to confirm.", + "auto-commit-to-branch": "auto-commit to {{ branch }} branch", + "default-create-entity-version-name": "{{entityName}} update", + "sync-strategy-merge-hint": "Creates or updates selected entities in the repository. All other repository entities are not modified.", + "sync-strategy-overwrite-hint": "Creates or updates selected entities in the repository. All other repository entities are deleted.", + "device-credentials-conflict": "Failed to load the device with external id {{entityId}}
due to the same credentials are already present in the database for another device.
Please consider disabling the load credentials setting in the restore form.", + "missing-referenced-entity": "Failed to load the {{sourceEntityTypeName}} with external id {{sourceEntityId}}
because it references missing {{targetEntityTypeName}} with id {{targetEntityId}}." + }, "widget": { "widget-library": "Widgets Library", "widget-bundle": "Widgets Bundle", diff --git a/ui-ngx/src/assets/locale/locale.constant-es_ES.json b/ui-ngx/src/assets/locale/locale.constant-es_ES.json index 56b85eac62..a684aaa568 100644 --- a/ui-ngx/src/assets/locale/locale.constant-es_ES.json +++ b/ui-ngx/src/assets/locale/locale.constant-es_ES.json @@ -59,7 +59,9 @@ "read-more": "Leer más", "hide": "Ocultar", "done": "Terminado", - "print": "Imprimir" + "print": "Imprimir", + "restore": "Restaurar", + "confirm": "Confirmar" }, "aggregation": { "aggregation": "Agrupación", @@ -322,6 +324,30 @@ "queue-submit-strategy": "Estrategia de envíos", "queue-processing-strategy": "Estrategia de procesamiento", "queue-configuration": "Configuración Cola", + "repository-settings": "Ajustes del repositorio", + "repository-url": "URL del repositorio", + "repository-url-required": "Se requiere la URL del repositorio.", + "default-branch": "Nombre de rama por defecto", + "authentication-settings": "Ajustes de autenticación", + "auth-method": "Método de autenticación", + "auth-method-username-password": "Usuario / Contraseña", + "auth-method-private-key": "Clave privada", + "password-access-token": "Contraseña / Tóken de acceso", + "change-password-access-token": "Cambiar contraseña / Tóken de acceso", + "private-key": "Clave privada", + "drop-private-key-file-or": "Arrastrar y soltar un fichero de llave privada o", + "passphrase": "Frase de contraseña", + "enter-passphrase": "Entrar frase de contraseña", + "change-passphrase": "Cambiar frase de contraseña", + "check-access": "Verificar acceso", + "check-repository-access-success": "Acceso al repositorio verificado satisfactoriamente!", + "delete-repository-settings-title": "Estás seguro de borrar los ajustes del repositorio?", + "delete-repository-settings-text": "Atención, tras la confirmación los ajustes del repositorio serán eliminados y la característica de control de versiones no estará disponible.", + "auto-commit-settings": "Ajustes Auto-publicar", + "auto-commit-entities": "Entidades Auto-publicar", + "no-auto-commit-entities-prompt": "No hay entidades configuradas para auto-publicar", + "delete-auto-commit-settings-title": "Estás seguro de borrar los ajustes de auto-publicar?", + "delete-auto-commit-settings-text": "Atención, tras la confirmación los ajustes de auto-publicar serán borrados y la característica de auto-publicar se desactivará para todas las entidades.", "2fa": { "2fa": "Autenticación de dos factores (2FA)", "available-providers": "Proveedores disponibles", @@ -485,7 +511,7 @@ "management": "Gestión de Activos", "view-assets": "Ver Activos", "add": "Añadir Activo", - + "asset-type-max-length": "El tipo de activo debe ser menor de 256", "assign-to-customer": "Asignar a cliente", "assign-asset-to-customer": "Asignar Activo(s) A Cliente", "assign-asset-to-customer-text": "Selecciona los activos a asignar al cliente", @@ -937,6 +963,8 @@ "assignedToCustomer": "Asignado al cliente", "assignedToCustomers": "Asignado a los clientes", "public": "Público", + "copyId": "Copiar ID del panel", + "idCopiedMessage": "El ID del panel ha sido copiado al portapapeles", "public-link": "Link público", "copy-public-link": "Copiar link público", "public-link-copied-message": "El link público del panel se ha copiado al portapapeles", @@ -1817,6 +1845,9 @@ "type-current-tenant": "Propietario Actual", "type-current-user": "Usuario Actual", "type-current-user-owner": "Usuario Propietario Actual", + "type-widgets-bundle": "Paquete de widgets", + "type-widgets-bundles": "Paquetes de widgets", + "list-of-widgets-bundles": "{ count, plural, 1 {Un paquete de widget} other {Lista de # paquetes de widgets} }", "search": "Buscar entidades", "selected-entities": "{ count, plural, 1 {1 entidad} other {# entidades} } seleccionadas", "entity-label": "Etiqueta de entidad", @@ -2246,6 +2277,7 @@ "current-device": "Dispositivo actual", "default-value": "Valor por defecto", "dynamic-source-type": "Tipo de origen dinámico", + "dynamic-value": "Valor dinámico", "no-dynamic-value": "Sin valor dinámico", "source-attribute": "Atributo de origen", "switch-to-dynamic-value": "Cambiar a valor dinámico", @@ -2488,7 +2520,7 @@ "request-password-reset": "Solicitar restablecer contraseña", "reset-password": "Restablecer contraseña", "create-password": "Crear contraseña", - "two-factor-authentication": "Two factor authentication", + "two-factor-authentication": "Autenticado de dos factores", "passwords-mismatch-error": "¡Las contraseñas introducidas deben ser iguales!", "password-again": "Repita la contraseña de nuevo", "sign-in": "Por favor, inicie sesión", @@ -2601,7 +2633,8 @@ "change-password": "Cambiar contraseña", "current-password": "Contraseña actual", "copy-jwt-token": "Copiar JWT", - "valid-till": "Válido hasta {{expirationData}}", + "jwt-token": "Tóken JWT", + "token-valid-till": "Válido hasta {{expirationData}}", "tokenCopiedSuccessMessage": "JWT copiado al portapapeles", "tokenCopiedWarnMessage": "JWT caducado, por favor actualiza la página." }, @@ -2660,6 +2693,20 @@ "backup-code-description": "Estos códigos de seguridad imprimibles son de un solo uso, te permiten identificarte cuando no tengas el teléfono a mano, útil cuando se viaja.", "backup-code-hint": "{{ info }} códigos de un solo uso activos en este momento" } + }, + "password-requirement": { + "at-least": "Al menos:", + "character": "{ count, plural, 1 {1 caracter} other {# caracteres} }", + "digit": "{ count, plural, 1 {1 dígito} other {# dígitos} }", + "incorrect-password-try-again": "Contraseña incorrecta. Prueba otra vez", + "lowercase-letter": "{ count, plural, 1 {1 minúscula} other {# mínusculas} }", + "new-passwords-not-match": "Las contraseñas no coinciden", + "password-should-not-contain-spaces": "La contraseña no puede contener espacios", + "password-not-meet-requirements": "La contraseña no reune los requisitos necesarios", + "password-requirements": "Requisitos de contraseña", + "password-should-difference": "La nueva contraseña debe ser diferente a la actual", + "special-character": "{ count, plural, 1 {1 caracter especial} other {# caracteres especiales} }", + "uppercase-letter": "{ count, plural, 1 {1 mayúscula} other {# mayúsculas} }" } }, "relation": { @@ -3053,6 +3100,8 @@ "transport-device-msg-rate-limit": "Tasa de mensajes de dispositivo.", "transport-device-telemetry-msg-rate-limit": "Tasa de mensajes de telemetría de dispositivo.", "transport-device-telemetry-data-points-rate-limit": "Tasa de datapoints de telemetría de dispositivo.", + "tenant-entity-export-rate-limit": "Tasa de creación de versión de entidades", + "tenant-entity-import-rate-limit": "Tasa de carga de versión de entidades", "max-transport-messages": "Nº Máximo de mensajes de transporte (0 - sin límite)", "max-transport-messages-required": "Nº Máximo de mensajes de transporte requerido.", "max-transport-messages-range": "Nº Máximo de mensajes de transporte no puede ser negativo", @@ -3091,7 +3140,22 @@ "max-created-alarms-range": "Nº Máximo de alarmas creadas no puede ser negativo", "no-queue": "No Queue configured", "add-queue": "Add Queue", - "queues-with-count": "Queues ({{count}})" + "queues-with-count": "Queues ({{count}})", + "tenant-rest-limits": "Rate limit for REST requests for tenant", + "customer-rest-limits": "Rate limit for REST requests for customer", + "incorrect-pattern-for-rate-limits": "The format is comma separated pairs of capacity and period (in seconds) with a colon between, e.g. 100:1,2000:60", + "too-small-value-zero": "The value must be bigger than 0", + "too-small-value-one": "The value must be bigger than 1", + "cassandra-tenant-limits-configuration": "Cassandra query rate limit for tenant", + "ws-limit-max-sessions-per-tenant": "Maximum number of WS sessions per tenant", + "ws-limit-max-sessions-per-customer": "Maximum number of WS sessions per customer", + "ws-limit-max-sessions-per-public-user": "Maximum number of WS sessions per public user", + "ws-limit-queue-per-session": "Maximum size of WS message queue per session", + "ws-limit-max-subscriptions-per-tenant": "Maximum number of WS subscriptions per tenant", + "ws-limit-max-subscriptions-per-customer": "Maximum number of WS subscriptions per customer", + "ws-limit-max-subscriptions-per-regular-user": "Maximum number of WS subscriptions per regular user", + "ws-limit-max-subscriptions-per-public-user": "Maximum number of WS subscriptions per public user", + "ws-limit-updates-per-session": "Rate limit for WS updates per session" }, "timeinterval": { "seconds-interval": "{ seconds, plural, 1 {1 segundo} other {# segundos} }", @@ -3220,6 +3284,68 @@ "json-value-invalid": "El valor JSON tiene un formato inválido", "json-value-required": "Se requiere valor JSON" }, + "version-control": { + "version-control": "Control de Versión", + "management": "Administrador de versiones", + "branch": "Rama", + "default": "Por defecto", + "select-branch": "Seleccionar rama", + "branch-required": "Se requiere rama", + "create-entity-version": "Versión creación de entidad", + "version-name": "Nombre de versión", + "version-name-required": "Se requiere nombre de versión", + "author": "Autor", + "export-relations": "Exportar relaciones", + "export-attributes": "Exportar atributos", + "export-credentials": "Exportar credenciales", + "entity-versions": "Versiones de entidad", + "versions": "Versiones", + "created-time": "Hora de creación", + "version-id": "ID de versión", + "no-entity-versions-text": "No se han encontrado versiones de entidad", + "no-versions-text": "No se han encontrado versiones", + "copy-full-version-id": "Copiar el ID de versión", + "create-version": "Crear versión", + "nothing-to-commit": "No hay cambios a publicar", + "restore-version": "Restaurar versión", + "restore-entity-from-version": "Restaurar entidad desde versión '{{versionName}}'", + "load-relations": "Cargar relaciones", + "load-attributes": "Cargar atributos", + "load-credentials": "Cargar credencialess", + "show-version-diff": "Mostrar diff de versión", + "diff-entity-with-version": "Diff de la versión de entidad '{{versionName}}'", + "previous-difference": "Anterior diferencia", + "next-difference": "Siguiente diferencia", + "current": "Actual", + "differences": "{ count, plural, 1 {1 diferencia} other {# diferencias} }", + "create-entities-version": "Crear versión de entidades", + "default-sync-strategy": "Estrategia de sincronización por defecto", + "sync-strategy-merge": "Combinar (Merge)", + "sync-strategy-overwrite": "Sobreescribir", + "entities-to-export": "Entidades a exportar", + "entities-to-restore": "Entidades a restaurar", + "sync-strategy": "Estrategia de sincronización", + "all-entities": "Todas las entidades", + "no-entities-to-export-prompt": "Por favor, especifica las entidades a exportar", + "no-entities-to-restore-prompt": "Por favor, especifica las entidades a restaurar", + "add-entity-type": "Añadir tipo de entidad", + "remove-all": "Borrar todo", + "version-create-result": "{ added, plural, 0 {Ninguna entidad} 1 {1 entidad} other {# entidades} } añadidas.
{ modified, plural, 0 {Ninguna entidad} 1 {1 entidad} other {# entidades} } modificadas.
{ removed, plural, 0 {Ninguna entidad} 1 {1 entidad} other {# entidades} } borradas.", + "remove-other-entities": "Borrar otras entidades", + "find-existing-entity-by-name": "Buscar entidad existente por nombre", + "restore-entities-from-version": "Restaurar entidades desde la versión '{{versionName}}'", + "no-entities-restored": "No se restauraron entidades", + "created": "{{created}} creadas", + "updated": "{{updated}} actualizadas", + "deleted": "{{deleted}} borradas", + "remove-other-entities-confirm-text": "Atención! Esta acción borrará permanentemente todas las entidades actuales
no presentes en la versión a restaurar.

Escribe remove other entities para confirmar.", + "auto-commit-to-branch": "auto-publicar a la rama {{ branch }}", + "default-create-entity-version-name": "{{entityName}} actualización", + "sync-strategy-merge-hint": "Crea o actualiza las entidades seleccionadas en el repositorio. Las demás entidades no serán modificadas.", + "sync-strategy-overwrite-hint": "Crea o actualiza las entidades seleccionadas en el repositorio. Las demás entidades serán borradas.", + "device-credentials-conflict": "Fallo al cargar el dispositivo con ID externo {{entityId}}
debido a que las mismas credenciales están ya presentes en la base de datos para otro dispositivo.
Por favor, considera desactivar el ajuste cargar credenciales en el formulario de restauración.", + "missing-referenced-entity": "Fallo al cargar {{sourceEntityTypeName}} con ID externo {{sourceEntityId}}
porque hace referencia al tipo de entidad {{targetEntityTypeName}} con el ID {{targetEntityId}}." + }, "widget": { "widget-library": "Bibloteca de Widgets", "widget-bundle": "Paquetes de Widgets", @@ -4431,6 +4557,13 @@ "material-icons": "Iconos material-design", "show-all": "Mostrar todos los iconos" }, + "phone-input": { + "phone-input-label": "Número de teléfono", + "phone-input-required": "Número de teléfono requerido", + "phone-input-validation": "El número es inválido o erróneo", + "phone-input-pattern": "Número inválido. Debe cumplir el formato E.164, ej. {{phoneNumber}}", + "phone-input-hint": "Número en el formato E.164, ej. {{phoneNumber}}" + }, "custom": { "widget-action": { "action-cell-button": "Acción botón de celda", diff --git a/ui-ngx/src/styles.scss b/ui-ngx/src/styles.scss index b6bf5fdfd2..8c6fffabf1 100644 --- a/ui-ngx/src/styles.scss +++ b/ui-ngx/src/styles.scss @@ -1263,8 +1263,8 @@ mat-label { } } - .tb-fullscreen-dialog-gt-sm { - @media #{$mat-gt-sm} { + .tb-fullscreen-dialog-gt-xs { + @media #{$mat-gt-xs} { min-height: 100%; min-width: 100%; max-width: none !important; diff --git a/ui-ngx/yarn.lock b/ui-ngx/yarn.lock index 9d0ef325b7..9005658ea8 100644 --- a/ui-ngx/yarn.lock +++ b/ui-ngx/yarn.lock @@ -1751,6 +1751,11 @@ "@turf/helpers" "^6.5.0" "@turf/meta" "^6.5.0" +"@types/ace-diff@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@types/ace-diff/-/ace-diff-2.1.1.tgz#1c08919aae8f9c429fcb139dc564c89dd093cbee" + integrity sha512-O27fCo2Y0njNslOFSewyRhTyXfLhVhleEU5aTI6ZqFTKENJ8L/LA+Y+ZfcHsHTtwrTWjBXqORmqEHH6Qytqw1w== + "@types/canvas-gauges@^2.1.4": version "2.1.4" resolved "https://registry.yarnpkg.com/@types/canvas-gauges/-/canvas-gauges-2.1.4.tgz#063881264597d098e78cf5ad921e8ed20ae2ad16" @@ -2190,6 +2195,13 @@ ace-builds@^1.4.13: resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.4.13.tgz#186f42d3849ebcc6a48b93088a058489897514c1" integrity sha512-SOLzdaQkY6ecPKYRDDg+MY1WoGgXA34cIvYJNNoBMGGUswHmlauU2Hy0UL96vW0Fs/LgFbMUjD+6vqzWTldIYQ== +ace-diff@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/ace-diff/-/ace-diff-3.0.3.tgz#84f685ff3d0b1910539fc39259ac73d8b6581e28" + integrity sha512-CJaV9Oi6BWLWGL2Kj//h5BNXlRCRu1GYHPOT7o+ZSAuJv9PaL9FWr/cCf16IuSVDo7oj6VriO+qgoHR8G9McLA== + dependencies: + diff-match-patch "^1.0.5" + acorn-import-assertions@^1.7.6: version "1.8.0" resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.8.0.tgz#ba2b5939ce62c238db6d93d81c9b111b29b855e9"