Browse Source

Merge remote-tracking branch 'origin/develop/3.4' into feature/edge-converters-integration

pull/6600/head
Volodymyr Babak 4 years ago
parent
commit
17ec3f41f4
  1. 4
      application/pom.xml
  2. 4
      application/src/main/data/json/system/widget_bundles/charts.json
  3. 35
      application/src/main/data/upgrade/3.3.4/schema_update.sql
  4. 2
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  5. 67
      application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java
  6. 156
      application/src/main/java/org/thingsboard/server/controller/AdminController.java
  7. 4
      application/src/main/java/org/thingsboard/server/controller/AssetController.java
  8. 25
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  9. 2
      application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java
  10. 2
      application/src/main/java/org/thingsboard/server/controller/DashboardController.java
  11. 4
      application/src/main/java/org/thingsboard/server/controller/DeviceController.java
  12. 4
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  13. 368
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  14. 1
      application/src/main/java/org/thingsboard/server/controller/RuleChainController.java
  15. 67
      application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java
  16. 10
      application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java
  17. 76
      application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java
  18. 26
      application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java
  19. 4
      application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java
  20. 4
      application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java
  21. 4
      application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java
  23. 7
      application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java
  24. 3
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  25. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java
  26. 1
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java
  27. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java
  28. 3
      application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java
  29. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java
  30. 1
      application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java
  31. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java
  32. 38
      application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java
  34. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java
  35. 2
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  36. 41
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  37. 6
      application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java
  38. 6
      application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java
  39. 4
      application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java
  40. 6
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java
  41. 2
      application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java
  42. 53
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  43. 4
      application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java
  44. 115
      application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java
  45. 11
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  46. 13
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  47. 3
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  48. 4
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  49. 24
      application/src/main/java/org/thingsboard/server/service/rule/DefaultTbRuleChainService.java
  50. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java
  51. 27
      application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java
  52. 2
      application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java
  53. 3
      application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java
  54. 2
      application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java
  55. 1
      application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java
  56. 2
      application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java
  57. 166
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  58. 40
      application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java
  59. 210
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java
  60. 28
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java
  61. 42
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java
  62. 42
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java
  63. 52
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java
  64. 61
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java
  65. 184
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
  66. 59
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java
  67. 43
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java
  68. 43
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java
  69. 76
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java
  70. 59
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java
  71. 32
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java
  72. 10
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java
  73. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java
  74. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java
  75. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java
  76. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java
  77. 81
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
  78. 398
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  79. 84
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java
  80. 139
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
  81. 105
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
  82. 93
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
  83. 91
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java
  84. 20
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java
  85. 30
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java
  86. 152
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
  87. 106
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java
  88. 554
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  89. 526
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
  90. 72
      application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
  91. 75
      application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java
  92. 32
      application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java
  93. 80
      application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java
  94. 34
      application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java
  95. 36
      application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java
  96. 36
      application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java
  97. 30
      application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java
  98. 30
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java
  99. 37
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java
  100. 43
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java

4
application/pom.xml

@ -69,6 +69,10 @@
<groupId>org.thingsboard.common</groupId>
<artifactId>cluster-api</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.common</groupId>
<artifactId>version-control</artifactId>
</dependency>
<dependency>
<groupId>org.thingsboard.rule-engine</groupId>
<artifactId>rule-engine-components</artifactId>

4
application/src/main/data/json/system/widget_bundles/charts.json

File diff suppressed because one or more lines are too long

35
application/src/main/data/upgrade/3.3.4/schema_update.sql

@ -14,6 +14,40 @@
-- limitations under the License.
--
ALTER TABLE device
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE device_profile
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE asset
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE rule_chain
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE rule_node
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE dashboard
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE customer
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE widgets_bundle
ADD COLUMN IF NOT EXISTS external_id UUID;
ALTER TABLE entity_view
ADD COLUMN IF NOT EXISTS external_id UUID;
CREATE INDEX IF NOT EXISTS idx_device_external_id ON device(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_device_profile_external_id ON device_profile(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_asset_external_id ON asset(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_rule_chain_external_id ON rule_chain(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_rule_node_external_id ON rule_node(rule_chain_id, external_id);
CREATE INDEX IF NOT EXISTS idx_dashboard_external_id ON dashboard(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_customer_external_id ON customer(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_widgets_bundle_external_id ON widgets_bundle(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_entity_view_external_id ON entity_view(tenant_id, external_id);
CREATE INDEX IF NOT EXISTS idx_rule_node_type ON rule_node(type);
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080';
CREATE TABLE IF NOT EXISTS queue (
id uuid NOT NULL CONSTRAINT queue_pkey PRIMARY KEY,
created_time bigint NOT NULL,
@ -35,3 +69,4 @@ CREATE TABLE IF NOT EXISTS user_auth_settings (
user_id uuid UNIQUE NOT NULL CONSTRAINT fk_user_auth_settings_user_id REFERENCES tb_user(id),
two_fa_settings varchar
);

2
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -48,7 +48,7 @@ import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.audit.AuditLogService;

67
application/src/main/java/org/thingsboard/server/config/RateLimitProcessingFilter.java

@ -15,17 +15,21 @@
*/
package org.thingsboard.server.config;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.security.authentication.BadCredentialsException;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Component;
import org.springframework.web.filter.GenericFilterBean;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.exception.ThingsboardErrorResponseHandler;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -35,42 +39,40 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Slf4j
@Component
public class RateLimitProcessingFilter extends GenericFilterBean {
@Value("${server.rest.limits.tenant.enabled:false}")
private boolean perTenantLimitsEnabled;
@Value("${server.rest.limits.tenant.configuration:}")
private String perTenantLimitsConfiguration;
@Value("${server.rest.limits.customer.enabled:false}")
private boolean perCustomerLimitsEnabled;
@Value("${server.rest.limits.customer.configuration:}")
private String perCustomerLimitsConfiguration;
@Autowired
private ThingsboardErrorResponseHandler errorResponseHandler;
private ConcurrentMap<TenantId, TbRateLimits> perTenantLimits = new ConcurrentHashMap<>();
private ConcurrentMap<CustomerId, TbRateLimits> perCustomerLimits = new ConcurrentHashMap<>();
@Autowired
@Lazy
private TbTenantProfileCache tenantProfileCache;
private final ConcurrentMap<TenantId, TbRateLimits> perTenantLimits = new ConcurrentHashMap<>();
private final ConcurrentMap<CustomerId, TbRateLimits> perCustomerLimits = new ConcurrentHashMap<>();
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
SecurityUser user = getCurrentUser();
if (user != null && !user.isSystemAdmin()) {
if (perTenantLimitsEnabled) {
TbRateLimits rateLimits = perTenantLimits.computeIfAbsent(user.getTenantId(), id -> new TbRateLimits(perTenantLimitsConfiguration));
if (!rateLimits.tryConsume()) {
errorResponseHandler.handle(new TbRateLimitsException(EntityType.TENANT), (HttpServletResponse) response);
return;
}
var profile = tenantProfileCache.get(user.getTenantId());
if (profile == null) {
log.debug("[{}] Failed to lookup tenant profile", user.getTenantId());
errorResponseHandler.handle(new BadCredentialsException("Failed to lookup tenant profile"), (HttpServletResponse) response);
return;
}
var profileConfiguration = profile.getDefaultProfileConfiguration();
if (!checkRateLimits(user.getTenantId(), profileConfiguration.getTenantServerRestLimitsConfiguration(), perTenantLimits, response)) {
return;
}
if (perCustomerLimitsEnabled && user.isCustomerUser()) {
TbRateLimits rateLimits = perCustomerLimits.computeIfAbsent(user.getCustomerId(), id -> new TbRateLimits(perCustomerLimitsConfiguration));
if (!rateLimits.tryConsume()) {
errorResponseHandler.handle(new TbRateLimitsException(EntityType.CUSTOMER), (HttpServletResponse) response);
if (user.isCustomerUser()) {
if (!checkRateLimits(user.getCustomerId(), profileConfiguration.getCustomerServerRestLimitsConfiguration(), perCustomerLimits, response)) {
return;
}
}
@ -78,6 +80,25 @@ public class RateLimitProcessingFilter extends GenericFilterBean {
chain.doFilter(request, response);
}
private <I extends EntityId> boolean checkRateLimits(I ownerId, String rateLimitConfig, Map<I, TbRateLimits> rateLimitsMap, ServletResponse response) {
if (StringUtils.isNotEmpty(rateLimitConfig)) {
TbRateLimits rateLimits = rateLimitsMap.get(ownerId);
if (rateLimits == null || !rateLimits.getConfiguration().equals(rateLimitConfig)) {
rateLimits = new TbRateLimits(rateLimitConfig);
rateLimitsMap.put(ownerId, rateLimits);
}
if (!rateLimits.tryConsume()) {
errorResponseHandler.handle(new TbRateLimitsException(ownerId.getEntityType()), (HttpServletResponse) response);
return false;
}
} else {
rateLimitsMap.remove(ownerId);
}
return true;
}
protected SecurityUser getCurrentUser() {
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
if (authentication != null && authentication.getPrincipal() instanceof SecurityUser) {

156
application/src/main/java/org/thingsboard/server/controller/AdminController.java

@ -16,16 +16,16 @@
package org.thingsboard.server.controller;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.rule.engine.api.MailService;
import org.thingsboard.rule.engine.api.SmsService;
import org.thingsboard.server.common.data.AdminSettings;
@ -34,14 +34,18 @@ import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.security.model.SecuritySettings;
import org.thingsboard.server.common.data.sms.config.TestSmsRequest;
import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.security.system.SystemSecurityService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService;
import org.thingsboard.server.service.update.UpdateService;
import static org.thingsboard.server.controller.ControllerConstants.SYSTEM_AUTHORITY_PARAGRAPH;
import static org.thingsboard.server.controller.ControllerConstants.*;
@RestController
@TbCoreComponent
@ -60,6 +64,12 @@ public class AdminController extends BaseController {
@Autowired
private SystemSecurityService systemSecurityService;
@Autowired
private EntitiesVersionControlService versionControlService;
@Autowired
private TbAutoCommitSettingsService autoCommitSettingsService;
@Autowired
private UpdateService updateService;
@ -96,6 +106,7 @@ public class AdminController extends BaseController {
@RequestBody AdminSettings adminSettings) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE);
adminSettings.setTenantId(getTenantId());
adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings));
if (adminSettings.getKey().equals("mail")) {
mailService.updateMailConfiguration();
@ -180,6 +191,137 @@ public class AdminController extends BaseController {
}
}
@ApiOperation(value = "Get repository settings (getRepositorySettings)",
notes = "Get the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/repositorySettings")
@ResponseBody
public RepositorySettings getRepositorySettings() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
RepositorySettings versionControlSettings = checkNotNull(versionControlService.getVersionControlSettings(getTenantId()));
versionControlSettings.setPassword(null);
versionControlSettings.setPrivateKey(null);
versionControlSettings.setPrivateKeyPassword(null);
return versionControlSettings;
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Check repository settings exists (repositorySettingsExists)",
notes = "Check whether the repository settings exists. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/repositorySettings/exists")
@ResponseBody
public Boolean repositorySettingsExists() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return versionControlService.getVersionControlSettings(getTenantId()) != null;
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Creates or Updates the repository settings (saveRepositorySettings)",
notes = "Creates or Updates the repository settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@PostMapping("/repositorySettings")
public DeferredResult<RepositorySettings> saveRepositorySettings(@RequestBody RepositorySettings settings) throws ThingsboardException {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE);
ListenableFuture<RepositorySettings> future = versionControlService.saveVersionControlSettings(getTenantId(), settings);
return wrapFuture(Futures.transform(future, savedSettings -> {
savedSettings.setPassword(null);
savedSettings.setPrivateKey(null);
savedSettings.setPrivateKeyPassword(null);
return savedSettings;
}, MoreExecutors.directExecutor()));
}
@ApiOperation(value = "Delete repository settings (deleteRepositorySettings)",
notes = "Deletes the repository settings."
+ TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/repositorySettings", method = RequestMethod.DELETE)
@ResponseStatus(value = HttpStatus.OK)
public DeferredResult<Void> deleteRepositorySettings() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.DELETE);
return wrapFuture(versionControlService.deleteVersionControlSettings(getTenantId()));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Check repository access (checkRepositoryAccess)",
notes = "Attempts to check repository access. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/repositorySettings/checkAccess", method = RequestMethod.POST)
public DeferredResult<Void> checkRepositoryAccess(
@ApiParam(value = "A JSON value representing the Repository Settings.")
@RequestBody RepositorySettings settings) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
settings = checkNotNull(settings);
return wrapFuture(versionControlService.checkVersionControlAccess(getTenantId(), settings));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Get auto commit settings (getAutoCommitSettings)",
notes = "Get the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/autoCommitSettings")
@ResponseBody
public AutoCommitSettings getAutoCommitSettings() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return checkNotNull(autoCommitSettingsService.get(getTenantId()));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Check auto commit settings exists (autoCommitSettingsExists)",
notes = "Check whether the auto commit settings exists. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@GetMapping("/autoCommitSettings/exists")
@ResponseBody
public Boolean autoCommitSettingsExists() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return autoCommitSettingsService.get(getTenantId()) != null;
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Creates or Updates the auto commit settings (saveAutoCommitSettings)",
notes = "Creates or Updates the auto commit settings object. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@PostMapping("/autoCommitSettings")
public AutoCommitSettings saveAutoCommitSettings(@RequestBody AutoCommitSettings settings) throws ThingsboardException {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE);
return autoCommitSettingsService.save(getTenantId(), settings);
}
@ApiOperation(value = "Delete auto commit settings (deleteAutoCommitSettings)",
notes = "Deletes the auto commit settings."
+ TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/autoCommitSettings", method = RequestMethod.DELETE)
@ResponseStatus(value = HttpStatus.OK)
public void deleteAutoCommitSettings() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.DELETE);
autoCommitSettingsService.delete(getTenantId());
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "Check for new Platform Releases (checkUpdates)",
notes = "Check notifications about new platform releases. "
+ SYSTEM_AUTHORITY_PARAGRAPH)

4
application/src/main/java/org/thingsboard/server/controller/AssetController.java

@ -51,9 +51,9 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.asset.AssetBulkImportService;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.service.entitiy.asset.TbAssetService;
import org.thingsboard.server.service.importing.BulkImportRequest;
import org.thingsboard.server.service.importing.BulkImportResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;

25
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -18,6 +18,10 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
@ -29,6 +33,7 @@ import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
@ -138,6 +143,7 @@ import org.thingsboard.server.service.security.permission.AccessControlService;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.state.DeviceStateService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.telemetry.AlarmSubscriptionService;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
@ -281,6 +287,9 @@ public abstract class BaseController {
@Autowired
protected QueueService queueService;
@Autowired
protected EntitiesVersionControlService vcService;
@Value("${server.log_controller_error_stack_trace}")
@Getter
private boolean logControllerErrorStackTrace;
@ -958,4 +967,20 @@ public abstract class BaseController {
return MediaType.APPLICATION_OCTET_STREAM;
}
}
protected <T> DeferredResult<T> wrapFuture(ListenableFuture<T> future) {
final DeferredResult<T> deferredResult = new DeferredResult<>();
Futures.addCallback(future, new FutureCallback<>() {
@Override
public void onSuccess(T result) {
deferredResult.setResult(result);
}
@Override
public void onFailure(Throwable t) {
deferredResult.setErrorResult(t);
}
}, MoreExecutors.directExecutor());
return deferredResult;
}
}

2
application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java

@ -135,6 +135,8 @@ public class ControllerConstants {
protected static final String EDGE_ASSIGN_ASYNC_FIRST_STEP_DESCRIPTION = "Assignment works in async way - first, notification event pushed to edge service queue on platform. ";
protected static final String EDGE_ASSIGN_RECEIVE_STEP_DESCRIPTION = "(Edge will receive this instantly, if it's currently connected, or once it's going to be connected to platform). ";
protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name.";
protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n";
protected static final String MARKDOWN_CODE_BLOCK_END = "\n```";
protected static final String EVENT_ERROR_FILTER_OBJ = MARKDOWN_CODE_BLOCK_START +

2
application/src/main/java/org/thingsboard/server/controller/DashboardController.java

@ -631,7 +631,7 @@ public class DashboardController extends BaseController {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.READ);
return tbDashboardService.asignDashboardToEdge(dashboardId, edge, getCurrentUser());
return tbDashboardService.assignDashboardToEdge(dashboardId, edge, getCurrentUser());
}
@ApiOperation(value = "Unassign dashboard from edge (unassignDashboardFromEdge)",

4
application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@ -66,9 +66,9 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.device.DeviceBulkImportService;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.service.entitiy.device.TbDeviceService;
import org.thingsboard.server.service.importing.BulkImportRequest;
import org.thingsboard.server.service.importing.BulkImportResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;

4
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -52,8 +52,8 @@ import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.edge.EdgeBulkImportService;
import org.thingsboard.server.service.entitiy.edge.TbEdgeService;
import org.thingsboard.server.service.importing.BulkImportRequest;
import org.thingsboard.server.service.importing.BulkImportResult;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;

368
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@ -0,0 +1,368 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.controller;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES;
@RestController
@TbCoreComponent
@RequestMapping("/api/entities/vc")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequiredArgsConstructor
public class EntitiesVersionControlController extends BaseController {
private final EntitiesVersionControlService versionControlService;
@ApiOperation(value = "", notes = "" +
"SINGLE_ENTITY:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"SINGLE_ENTITY\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
" \"branch\": \"dev\",\n" +
"\n" +
" \"entityId\": {\n" +
" \"entityType\": \"DEVICE\",\n" +
" \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" },\n" +
" \"config\": {\n" +
" \"saveRelations\": true\n" +
" }\n" +
"}\n```" + NEW_LINE +
"COMPLEX:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"COMPLEX\",\n" +
"\n" +
" \"versionName\": \"Devices and profiles: release 2\",\n" +
" \"branch\": \"master\",\n" +
"\n" +
" \"syncStrategy\": \"OVERWRITE\",\n" +
" \"entityTypes\": {\n" +
" \"DEVICE\": {\n" +
" \"syncStrategy\": null,\n" +
" \"allEntities\": true,\n" +
" \"saveRelations\": true\n" +
" },\n" +
" \"DEVICE_PROFILE\": {\n" +
" \"syncStrategy\": \"MERGE\",\n" +
" \"allEntities\": false,\n" +
" \"entityIds\": [\n" +
" \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" ],\n" +
" \"saveRelations\": true\n" +
" }\n" +
" }\n" +
"}\n```")
@PostMapping("/version")
public DeferredResult<VersionCreationResult> saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws ThingsboardException {
SecurityUser user = getCurrentUser();
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.WRITE);
return wrapFuture(versionControlService.saveEntitiesVersion(user, request));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" +
" \"name\": \"Device profile 1 version 1.0\"\n" +
" }\n" +
"]\n```")
@GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page,
@ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION)
@RequestParam(required = false) String textSearch,
@ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp")
@RequestParam(required = false) String sortProperty,
@ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES)
@RequestParam(required = false) String sortOrder) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
EntityId externalEntityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid);
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" +
" \"name\": \"Device profiles from dev\"\n" +
" }\n" +
"]\n```")
@GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page,
@ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION)
@RequestParam(required = false) String textSearch,
@ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp")
@RequestParam(required = false) String sortProperty,
@ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES)
@RequestParam(required = false) String sortOrder) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" +
" \"name\": \"Device 1 removed\"\n" +
" },\n" +
" {\n" +
" \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" +
" \"name\": \"Device profiles added\"\n" +
" },\n" +
" {\n" +
" \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" +
" \"name\": \"Devices added\"\n" +
" }\n" +
"]\n```")
@GetMapping(value = "/version/{branch}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@PathVariable String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@RequestParam int page,
@ApiParam(value = ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION)
@RequestParam(required = false) String textSearch,
@ApiParam(value = SORT_PROPERTY_DESCRIPTION, allowableValues = "timestamp")
@RequestParam(required = false) String sortProperty,
@ApiParam(value = SORT_ORDER_DESCRIPTION, allowableValues = SORT_ORDER_ALLOWABLE_VALUES)
@RequestParam(required = false) String sortOrder) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
return wrapFuture(versionControlService.listVersions(getTenantId(), branch, pageLink));
} catch (Exception e) {
throw handleException(e);
}
}
@GetMapping("/entity/{branch}/{entityType}/{versionId}")
public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@PathVariable String branch,
@PathVariable EntityType entityType,
@PathVariable String versionId) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType));
} catch (Exception e) {
throw handleException(e);
}
}
@GetMapping("/entity/{branch}/{versionId}")
public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@PathVariable String branch,
@PathVariable String versionId) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId));
} catch (Exception e) {
throw handleException(e);
}
}
@GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}")
public DeferredResult<EntityDataInfo> getEntityDataInfo(@PathVariable String versionId,
@PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid);
return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId));
} catch (Exception e) {
throw handleException(e);
}
}
@GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}")
public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@PathVariable String branch,
@PathVariable EntityType entityType,
@PathVariable UUID internalEntityUuid,
@RequestParam String versionId) throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid);
return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "", notes = "" +
"SINGLE_ENTITY:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"SINGLE_ENTITY\",\n" +
" \n" +
" \"branch\": \"dev\",\n" +
" \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" +
" \n" +
" \"externalEntityId\": {\n" +
" \"entityType\": \"DEVICE\",\n" +
" \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" +
" },\n" +
" \"config\": {\n" +
" \"loadRelations\": false,\n" +
" \"findExistingEntityByName\": false\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_TYPE:" + NEW_LINE +
"```\n{\n" +
" \"type\": \"ENTITY_TYPE\",\n" +
"\n" +
" \"branch\": \"dev\",\n" +
" \"versionId\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" +
"\n" +
" \"entityTypes\": {\n" +
" \"DEVICE\": {\n" +
" \"loadRelations\": false,\n" +
" \"findExistingEntityByName\": false,\n" +
" \"removeOtherEntities\": true\n" +
" }\n" +
" }\n" +
"}\n```")
@PostMapping("/entity")
public DeferredResult<VersionLoadResult> loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws ThingsboardException {
SecurityUser user = getCurrentUser();
try {
accessControlService.checkPermission(user, Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.loadEntitiesVersion(user, request));
} catch (Exception e) {
throw handleException(e);
}
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"name\": \"master\",\n" +
" \"default\": true\n" +
" },\n" +
" {\n" +
" \"name\": \"dev\",\n" +
" \"default\": false\n" +
" },\n" +
" {\n" +
" \"name\": \"dev-2\",\n" +
" \"default\": false\n" +
" }\n" +
"]\n\n```")
@GetMapping("/branches")
public DeferredResult<List<BranchInfo>> listBranches() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
final TenantId tenantId = getTenantId();
ListenableFuture<List<String>> branches = versionControlService.listBranches(tenantId);
return wrapFuture(Futures.transform(branches, remoteBranches -> {
List<BranchInfo> infos = new ArrayList<>();
String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch();
if (StringUtils.isNotEmpty(defaultBranch)) {
infos.add(new BranchInfo(defaultBranch, true));
}
remoteBranches.forEach(branch -> {
if (!branch.equals(defaultBranch)) {
infos.add(new BranchInfo(branch, false));
}
});
return infos;
}, MoreExecutors.directExecutor()));
} catch (Exception e) {
throw handleException(e);
}
}
@Data
public static class BranchInfo {
private final String name;
private final boolean isDefault;
}
}

1
application/src/main/java/org/thingsboard/server/controller/RuleChainController.java

@ -233,7 +233,6 @@ public class RuleChainController extends BaseController {
public RuleChain saveRuleChain(
@ApiParam(value = "A JSON value representing the rule chain.")
@RequestBody RuleChain ruleChain) throws ThingsboardException {
ruleChain.setTenantId(getCurrentUser().getTenantId());
checkEntity(ruleChain.getId(), ruleChain, Resource.RULE_CHAIN);
return tbRuleChainService.save(ruleChain, getCurrentUser());

67
application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java

@ -16,12 +16,12 @@
package org.thingsboard.server.controller.plugin;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.BeanCreationNotAllowedException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.core.Authentication;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.PongMessage;
import org.springframework.web.socket.TextMessage;
@ -34,10 +34,10 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.config.WebSocketConfiguration;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.telemetry.DefaultTelemetryWebSocketService;
import org.thingsboard.server.service.telemetry.SessionEvent;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketMsgEndpoint;
import org.thingsboard.server.service.telemetry.TelemetryWebSocketService;
@ -72,22 +72,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
@Autowired
private TelemetryWebSocketService webSocketService;
@Autowired
private TbTenantProfileCache tenantProfileCache;
@Value("${server.ws.send_timeout:5000}")
private long sendTimeout;
@Value("${server.ws.limits.max_sessions_per_tenant:0}")
private int maxSessionsPerTenant;
@Value("${server.ws.limits.max_sessions_per_customer:0}")
private int maxSessionsPerCustomer;
@Value("${server.ws.limits.max_sessions_per_regular_user:0}")
private int maxSessionsPerRegularUser;
@Value("${server.ws.limits.max_sessions_per_public_user:0}")
private int maxSessionsPerPublicUser;
@Value("${server.ws.limits.max_queue_per_ws_session:1000}")
private int maxMsgQueuePerSession;
@Value("${server.ws.limits.max_updates_per_session:}")
private String perSessionUpdatesConfiguration;
@Value("${server.ws.ping_timeout:30000}")
private long pingTimeout;
@ -144,10 +133,13 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
String internalSessionId = session.getId();
TelemetryWebSocketSessionRef sessionRef = toRef(session);
String externalSessionId = sessionRef.getSessionId();
if (!checkLimits(session, sessionRef)) {
return;
}
internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, maxMsgQueuePerSession));
var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration();
internalSessionMap.put(internalSessionId, new SessionMetaData(session, sessionRef, tenantProfileConfiguration.getWsMsgQueueLimitPerSession() > 0 ?
tenantProfileConfiguration.getWsMsgQueueLimitPerSession() : 500));
externalSessionMap.put(externalSessionId, internalSessionId);
processInWebSocketService(sessionRef, SessionEvent.onEstablished());
@ -323,8 +315,9 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
if (internalId != null) {
SessionMetaData sessionMd = internalSessionMap.get(internalId);
if (sessionMd != null) {
if (!StringUtils.isEmpty(perSessionUpdatesConfiguration)) {
TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(perSessionUpdatesConfiguration));
var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration();
if (StringUtils.isNotEmpty(tenantProfileConfiguration.getWsUpdatesPerSessionRateLimit())) {
TbRateLimits rateLimits = perSessionUpdateLimits.computeIfAbsent(sessionRef.getSessionId(), sid -> new TbRateLimits(tenantProfileConfiguration.getWsUpdatesPerSessionRateLimit()));
if (!rateLimits.tryConsume()) {
if (blacklistedSessions.putIfAbsent(externalId, sessionRef) == null) {
log.info("[{}][{}][{}] Failed to process session update. Max session updates limit reached"
@ -336,6 +329,8 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
log.debug("[{}][{}][{}] Session is no longer blacklisted.", sessionRef.getSecurityCtx().getTenantId(), sessionRef.getSecurityCtx().getId(), externalId);
blacklistedSessions.remove(externalId);
}
} else {
perSessionUpdateLimits.remove(sessionRef.getSessionId());
}
sessionMd.sendMsg(msg);
} else {
@ -380,11 +375,17 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
private boolean checkLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) throws Exception {
var tenantProfileConfiguration =
tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration();
if (tenantProfileConfiguration == null) {
return true;
}
String sessionId = session.getId();
if (maxSessionsPerTenant > 0) {
if (tenantProfileConfiguration.getMaxWsSessionsPerTenant() > 0) {
Set<String> tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet());
synchronized (tenantSessions) {
if (tenantSessions.size() < maxSessionsPerTenant) {
if (tenantSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerTenant()) {
tenantSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max tenant sessions limit reached"
@ -396,10 +397,10 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
if (sessionRef.getSecurityCtx().isCustomerUser()) {
if (maxSessionsPerCustomer > 0) {
if (tenantProfileConfiguration.getMaxWsSessionsPerCustomer() > 0) {
Set<String> customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet());
synchronized (customerSessions) {
if (customerSessions.size() < maxSessionsPerCustomer) {
if (customerSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerCustomer()) {
customerSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max customer sessions limit reached"
@ -409,10 +410,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
}
}
if (maxSessionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
if (tenantProfileConfiguration.getMaxWsSessionsPerRegularUser() > 0
&& UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (regularUserSessions) {
if (regularUserSessions.size() < maxSessionsPerRegularUser) {
if (regularUserSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerRegularUser()) {
regularUserSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max regular user sessions limit reached"
@ -422,10 +424,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
}
}
if (maxSessionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
if (tenantProfileConfiguration.getMaxWsSessionsPerPublicUser() > 0
&& UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (publicUserSessions) {
if (publicUserSessions.size() < maxSessionsPerPublicUser) {
if (publicUserSessions.size() < tenantProfileConfiguration.getMaxWsSessionsPerPublicUser()) {
publicUserSessions.add(sessionId);
} else {
log.info("[{}][{}][{}] Failed to start session. Max public user sessions limit reached"
@ -440,29 +443,31 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements Telemetr
}
private void cleanupLimits(WebSocketSession session, TelemetryWebSocketSessionRef sessionRef) {
var tenantProfileConfiguration = tenantProfileCache.get(sessionRef.getSecurityCtx().getTenantId()).getDefaultProfileConfiguration();
String sessionId = session.getId();
perSessionUpdateLimits.remove(sessionRef.getSessionId());
blacklistedSessions.remove(sessionRef.getSessionId());
if (maxSessionsPerTenant > 0) {
if (tenantProfileConfiguration.getMaxWsSessionsPerTenant() > 0) {
Set<String> tenantSessions = tenantSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getTenantId(), id -> ConcurrentHashMap.newKeySet());
synchronized (tenantSessions) {
tenantSessions.remove(sessionId);
}
}
if (sessionRef.getSecurityCtx().isCustomerUser()) {
if (maxSessionsPerCustomer > 0) {
if (tenantProfileConfiguration.getMaxWsSessionsPerCustomer() > 0) {
Set<String> customerSessions = customerSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getCustomerId(), id -> ConcurrentHashMap.newKeySet());
synchronized (customerSessions) {
customerSessions.remove(sessionId);
}
}
if (maxSessionsPerRegularUser > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
if (tenantProfileConfiguration.getMaxWsSessionsPerRegularUser() > 0 && UserPrincipal.Type.USER_NAME.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> regularUserSessions = regularUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (regularUserSessions) {
regularUserSessions.remove(sessionId);
}
}
if (maxSessionsPerPublicUser > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
if (tenantProfileConfiguration.getMaxWsSessionsPerPublicUser() > 0 && UserPrincipal.Type.PUBLIC_ID.equals(sessionRef.getSecurityCtx().getUserPrincipal().getType())) {
Set<String> publicUserSessions = publicUserSessionsMap.computeIfAbsent(sessionRef.getSecurityCtx().getId(), id -> ConcurrentHashMap.newKeySet());
synchronized (publicUserSessions) {
publicUserSessions.remove(sessionId);

10
application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java

@ -22,13 +22,14 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
@ -41,7 +42,6 @@ import org.thingsboard.server.common.msg.TbMsgDataType;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.dao.audit.AuditLogService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.cluster.TbClusterService;
import java.util.List;
import java.util.Map;
@ -212,7 +212,7 @@ public class EntityActionService {
}
}
public <E extends HasName, I extends EntityId> void logEntityAction(User user, I entityId, E entity, CustomerId customerId,
public <E extends HasName, I extends EntityId> void logEntityAction(User user, I entityId, E entity, CustomerId customerId,
ActionType actionType, Exception e, Object... additionalInfo) {
if (customerId == null || customerId.isNullUid()) {
customerId = user.getCustomerId();
@ -223,6 +223,9 @@ public class EntityActionService {
auditLogService.logEntityAction(user.getTenantId(), customerId, user.getId(), user.getName(), entityId, entity, actionType, e, additionalInfo);
}
public void sendEntityNotificationMsgToEdgeService(TenantId tenantId, EntityId entityId, EdgeEventActionType action) {
tbClusterService.sendNotificationMsgToEdgeService(tenantId, null, entityId, null, null, action);
}
private <T> T extractParameter(Class<T> clazz, int index, Object... additionalInfo) {
T result = null;
@ -267,4 +270,5 @@ public class EntityActionService {
entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
}
}
}

76
application/src/main/java/org/thingsboard/server/service/apiusage/DefaultRateLimitService.java

@ -0,0 +1,76 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.msg.tools.TbRateLimits;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
@Service
@RequiredArgsConstructor
public class DefaultRateLimitService implements RateLimitService {
private final TbTenantProfileCache tenantProfileCache;
private final Map<String, Map<TenantId, TbRateLimits>> rateLimits = new ConcurrentHashMap<>();
@Override
public boolean checkEntityExportLimit(TenantId tenantId) {
return checkLimit(tenantId, "entityExport", DefaultTenantProfileConfiguration::getTenantEntityExportRateLimit);
}
@Override
public boolean checkEntityImportLimit(TenantId tenantId) {
return checkLimit(tenantId, "entityImport", DefaultTenantProfileConfiguration::getTenantEntityImportRateLimit);
}
private boolean checkLimit(TenantId tenantId, String rateLimitsKey, Function<DefaultTenantProfileConfiguration, String> rateLimitConfigExtractor) {
String rateLimitConfig = tenantProfileCache.get(tenantId).getProfileConfiguration()
.map(rateLimitConfigExtractor).orElse(null);
Map<TenantId, TbRateLimits> rateLimits = this.rateLimits.get(rateLimitsKey);
if (StringUtils.isEmpty(rateLimitConfig)) {
if (rateLimits != null) {
rateLimits.remove(tenantId);
if (rateLimits.isEmpty()) {
this.rateLimits.remove(rateLimitsKey);
}
}
return true;
}
if (rateLimits == null) {
rateLimits = new ConcurrentHashMap<>();
this.rateLimits.put(rateLimitsKey, rateLimits);
}
TbRateLimits rateLimit = rateLimits.get(tenantId);
if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) {
rateLimit = new TbRateLimits(rateLimitConfig);
rateLimits.put(tenantId, rateLimit);
}
return rateLimit.tryConsume();
}
}

26
application/src/main/java/org/thingsboard/server/service/apiusage/RateLimitService.java

@ -0,0 +1,26 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.apiusage;
import org.thingsboard.server.common.data.id.TenantId;
public interface RateLimitService {
boolean checkEntityExportLimit(TenantId tenantId);
boolean checkEntityImportLimit(TenantId tenantId);
}

4
application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java

@ -26,9 +26,9 @@ import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType;
import org.thingsboard.server.service.entitiy.asset.TbAssetService;
import org.thingsboard.server.service.importing.AbstractBulkImportService;
import org.thingsboard.server.service.importing.BulkImportColumnType;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.Map;

4
application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java

@ -49,9 +49,9 @@ import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType;
import org.thingsboard.server.service.entitiy.device.TbDeviceService;
import org.thingsboard.server.service.importing.AbstractBulkImportService;
import org.thingsboard.server.service.importing.BulkImportColumnType;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.Collection;

4
application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java

@ -28,9 +28,9 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService;
import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType;
import org.thingsboard.server.service.entitiy.edge.TbEdgeService;
import org.thingsboard.server.service.importing.AbstractBulkImportService;
import org.thingsboard.server.service.importing.BulkImportColumnType;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.Map;

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/DeviceProfileMsgConstructor.java

@ -20,7 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.queue.util.TbCoreComponent;

7
application/src/main/java/org/thingsboard/server/service/edge/rpc/fetch/AdminSettingsEdgeEventFetcher.java

@ -83,7 +83,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailSettings)));
AdminSettings tenantMailSettings = convertToTenantAdminSettings(systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue());
AdminSettings tenantMailSettings = convertToTenantAdminSettings(tenantId, systemMailSettings.getKey(), (ObjectNode) systemMailSettings.getJsonValue());
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailSettings)));
@ -91,7 +91,7 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(systemMailTemplates)));
AdminSettings tenantMailTemplates = convertToTenantAdminSettings(systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue());
AdminSettings tenantMailTemplates = convertToTenantAdminSettings(tenantId, systemMailTemplates.getKey(), (ObjectNode) systemMailTemplates.getJsonValue());
result.add(EdgeUtils.constructEdgeEvent(tenantId, edge.getId(), EdgeEventType.ADMIN_SETTINGS,
EdgeEventActionType.UPDATED, null, mapper.valueToTree(tenantMailTemplates)));
@ -151,8 +151,9 @@ public class AdminSettingsEdgeEventFetcher implements EdgeEventFetcher {
}
}
private AdminSettings convertToTenantAdminSettings(String key, ObjectNode jsonValue) {
private AdminSettings convertToTenantAdminSettings(TenantId tenantId, String key, ObjectNode jsonValue) {
AdminSettings tenantMailSettings = new AdminSettings();
tenantMailSettings.setTenantId(tenantId);
jsonValue.put("useSystemMailSettings", true);
tenantMailSettings.setJsonValue(jsonValue);
tenantMailSettings.setKey(key);

3
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -64,6 +64,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.edge.EdgeNotificationService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.resource.TbResourceService;
@ -125,6 +126,8 @@ public abstract class AbstractTbEntityService {
@Autowired
protected DashboardService dashboardService;
@Autowired
protected EntitiesVersionControlService vcService;
@Autowired
protected EntityViewService entityViewService;
@Autowired
protected TelemetrySubscriptionService tsSubService;

2
application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java

@ -334,7 +334,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
return null;
}
private EdgeEventActionType edgeTypeByActionType(ActionType actionType) {
public static EdgeEventActionType edgeTypeByActionType(ActionType actionType) {
switch (actionType) {
case ADDED:
return EdgeEventActionType.ADDED;

1
application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java

@ -46,6 +46,7 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb
TenantId tenantId = asset.getTenantId();
try {
Asset savedAsset = checkNotNull(assetService.saveAsset(asset));
vcService.autoCommit(user, savedAsset.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), asset, savedAsset.getCustomerId(), actionType, user);
return savedAsset;
} catch (Exception e) {

4
application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java

@ -40,10 +40,10 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements
public Customer save(Customer customer, SecurityUser user) throws ThingsboardException {
ActionType actionType = customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = customer.getTenantId();
CustomerId customerId = customer.getId();
try {
Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer));
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, customerId, actionType, user);
vcService.autoCommit(user, savedCustomer.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user);
return savedCustomer;
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.CUSTOMER), customer, null, actionType, user, e);

3
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java

@ -48,6 +48,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement
TenantId tenantId = dashboard.getTenantId();
try {
Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard));
vcService.autoCommit(user, savedDashboard.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard,
null, actionType, user);
return savedDashboard;
@ -219,7 +220,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement
}
@Override
public Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException {
public Dashboard assignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.ASSIGNED_TO_EDGE;
TenantId tenantId = user.getTenantId();
EdgeId edgeId = edge.getId();

4
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java

@ -26,7 +26,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.Set;
public interface TbDashboardService extends SimpleTbEntityService<Dashboard> {
public interface TbDashboardService extends SimpleTbEntityService<Dashboard> {
Dashboard assignDashboardToCustomer(DashboardId dashboardId, Customer customer, SecurityUser user) throws ThingsboardException;
@ -40,7 +40,7 @@ public interface TbDashboardService extends SimpleTbEntityService<Dashboard> {
Dashboard removeDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException;
Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException;
Dashboard assignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException;
Dashboard unassignDashboardFromEdge(Dashboard dashboard, Edge edge, SecurityUser user) throws ThingsboardException;

1
application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java

@ -54,6 +54,7 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T
ActionType actionType = device.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
try {
Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken));
vcService.autoCommit(user, savedDevice.getId());
notificationEntityService.notifyCreateOrUpdateDevice(tenantId, savedDevice.getId(), savedDevice.getCustomerId(),
savedDevice, oldDevice, actionType, user);

2
application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java

@ -54,7 +54,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple
}
}
DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile));
vcService.autoCommit(user, savedDeviceProfile.getId());
tbClusterService.onDeviceProfileChange(savedDeviceProfile, null);
tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedDeviceProfile.getId(),
actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);

38
application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java

@ -66,19 +66,33 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
@Override
public EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException {
ActionType actionType = entityView.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
try {
EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView));
this.updateEntityViewAttributes(user, savedEntityView, existingEntityView);
notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView,
null, actionType, user);
return savedEntityView;
} catch (Exception e) {
notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e);
throw handleException(e);
}
}
@Override
public void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException {
try {
List<ListenableFuture<?>> futures = new ArrayList<>();
if (existingEntityView != null) {
if (existingEntityView.getKeys() != null && existingEntityView.getKeys().getAttributes() != null) {
futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.CLIENT_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user));
futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SERVER_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user));
futures.add(deleteAttributesFromEntityView(existingEntityView, DataConstants.SHARED_SCOPE, existingEntityView.getKeys().getAttributes().getCs(), user));
if (oldEntityView != null) {
if (oldEntityView.getKeys() != null && oldEntityView.getKeys().getAttributes() != null) {
futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.CLIENT_SCOPE, oldEntityView.getKeys().getAttributes().getCs(), user));
futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SERVER_SCOPE, oldEntityView.getKeys().getAttributes().getSs(), user));
futures.add(deleteAttributesFromEntityView(oldEntityView, DataConstants.SHARED_SCOPE, oldEntityView.getKeys().getAttributes().getSh(), user));
}
List<String> tsKeys = existingEntityView.getKeys() != null && existingEntityView.getKeys().getTimeseries() != null ?
existingEntityView.getKeys().getTimeseries() : Collections.emptyList();
futures.add(deleteLatestFromEntityView(existingEntityView, tsKeys, user));
List<String> tsKeys = oldEntityView.getKeys() != null && oldEntityView.getKeys().getTimeseries() != null ?
oldEntityView.getKeys().getTimeseries() : Collections.emptyList();
futures.add(deleteLatestFromEntityView(oldEntityView, tsKeys, user));
}
EntityView savedEntityView = checkNotNull(entityViewService.saveEntityView(entityView));
if (savedEntityView.getKeys() != null) {
if (savedEntityView.getKeys().getAttributes() != null) {
futures.add(copyAttributesFromEntityToEntityView(savedEntityView, DataConstants.CLIENT_SCOPE, savedEntityView.getKeys().getAttributes().getCs(), user));
@ -94,13 +108,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
throw new RuntimeException("Failed to copy attributes to entity view", e);
}
}
notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView,
null, actionType, user);
return savedEntityView;
} catch (Exception e) {
notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e);
throw handleException(e);
}
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java

@ -28,6 +28,8 @@ public interface TbEntityViewService {
EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException;
void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException;
void delete (EntityView entity, SecurityUser user) throws ThingsboardException;
EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer,

4
application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java

@ -27,8 +27,10 @@ import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.entitiy.queue.TbQueueService;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
@Service
@TbCoreComponent
@ -38,6 +40,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T
private final InstallScripts installScripts;
private final TbQueueService tbQueueService;
private final TenantProfileService tenantProfileService;
private final EntitiesVersionControlService versionControlService;
@Override
public Tenant save(Tenant tenant) throws ThingsboardException {
@ -70,6 +73,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T
tenantService.deleteTenant(tenantId);
tenantProfileCache.evict(tenantId);
notificationEntityService.notifyDeleteTenant(tenant);
versionControlService.deleteVersionControlSettings(tenantId).get(1, TimeUnit.MINUTES);
} catch (Exception e) {
throw handleException(e);
}

2
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -265,6 +265,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Override
public void createAdminSettings() throws Exception {
AdminSettings generalSettings = new AdminSettings();
generalSettings.setTenantId(TenantId.SYS_TENANT_ID);
generalSettings.setKey("general");
ObjectNode node = objectMapper.createObjectNode();
node.put("baseUrl", "http://localhost:8080");
@ -273,6 +274,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, generalSettings);
AdminSettings mailSettings = new AdminSettings();
mailSettings.setTenantId(TenantId.SYS_TENANT_ID);
mailSettings.setKey("mail");
node = objectMapper.createObjectNode();
node.put("mailFrom", "ThingsBoard <sysadmin@localhost.localdomain>");

41
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -571,6 +571,22 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
log.info("Loading queues...");
try {
if (!CollectionUtils.isEmpty(queueConfig.getQueues())) {
queueConfig.getQueues().forEach(queueSettings -> {
Queue queue = queueConfigToQueue(queueSettings);
Queue existing = queueService.findQueueByTenantIdAndName(queue.getTenantId(), queue.getName());
if (existing == null) {
queueService.saveQueue(queue);
}
});
} else {
systemDataLoaderService.createQueues();
}
} catch (Exception e) {
}
log.info("Updating device profiles...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", "schema_update_device_profile.sql");
loadSql(schemaUpdateFile, conn);
@ -628,4 +644,29 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
return isOldSchema;
}
private Queue queueConfigToQueue(TbRuleEngineQueueConfiguration queueSettings) {
Queue queue = new Queue();
queue.setTenantId(TenantId.SYS_TENANT_ID);
queue.setName(queueSettings.getName());
queue.setTopic(queueSettings.getTopic());
queue.setPollInterval(queueSettings.getPollInterval());
queue.setPartitions(queueSettings.getPartitions());
queue.setPackProcessingTimeout(queueSettings.getPackProcessingTimeout());
SubmitStrategy submitStrategy = new SubmitStrategy();
submitStrategy.setBatchSize(queueSettings.getSubmitStrategy().getBatchSize());
submitStrategy.setType(SubmitStrategyType.valueOf(queueSettings.getSubmitStrategy().getType()));
queue.setSubmitStrategy(submitStrategy);
ProcessingStrategy processingStrategy = new ProcessingStrategy();
processingStrategy.setType(ProcessingStrategyType.valueOf(queueSettings.getProcessingStrategy().getType()));
processingStrategy.setRetries(queueSettings.getProcessingStrategy().getRetries());
processingStrategy.setFailurePercentage(queueSettings.getProcessingStrategy().getFailurePercentage());
processingStrategy.setPauseBetweenRetries(queueSettings.getProcessingStrategy().getPauseBetweenRetries());
processingStrategy.setMaxPauseBetweenRetries(queueSettings.getProcessingStrategy().getMaxPauseBetweenRetries());
queue.setProcessingStrategy(processingStrategy);
queue.setConsumerPerPartition(queueSettings.isConsumerPerPartition());
return queue;
}
}

6
application/src/main/java/org/thingsboard/server/service/install/PsqlEntityDatabaseSchemaService.java → application/src/main/java/org/thingsboard/server/service/install/SqlEntityDatabaseSchemaService.java

@ -18,19 +18,17 @@ package org.thingsboard.server.service.install;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
@Service
@PsqlDao
@Profile("install")
@Slf4j
public class PsqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
public class SqlEntityDatabaseSchemaService extends SqlAbstractDatabaseSchemaService
implements EntityDatabaseSchemaService {
public static final String SCHEMA_ENTITIES_SQL = "schema-entities.sql";
public static final String SCHEMA_ENTITIES_IDX_SQL = "schema-entities-idx.sql";
public static final String SCHEMA_ENTITIES_IDX_PSQL_ADDON_SQL = "schema-entities-idx-psql-addon.sql";
public PsqlEntityDatabaseSchemaService() {
public SqlEntityDatabaseSchemaService() {
super(SCHEMA_ENTITIES_SQL, SCHEMA_ENTITIES_IDX_SQL);
}

6
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseSchemaService.java → application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseSchemaService.java

@ -18,19 +18,17 @@ package org.thingsboard.server.service.install;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.SqlTsDao;
@Service
@SqlTsDao
@PsqlDao
@Profile("install")
public class PsqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService {
public class SqlTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService {
@Value("${sql.postgres.ts_key_value_partitioning:MONTHS}")
private String partitionType;
public PsqlTsDatabaseSchemaService() {
public SqlTsDatabaseSchemaService() {
super("schema-ts-psql.sql", null);
}

4
application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java → application/src/main/java/org/thingsboard/server/service/install/SqlTsDatabaseUpgradeService.java

@ -21,7 +21,6 @@ import org.apache.commons.lang3.SystemUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.SqlTsDao;
import java.io.File;
@ -36,8 +35,7 @@ import java.sql.DriverManager;
@Profile("install")
@Slf4j
@SqlTsDao
@PsqlDao
public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
public class SqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
@Value("${sql.postgres.ts_key_value_partitioning:MONTHS}")
private String partitionType;

6
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseSchemaService.java

@ -19,16 +19,10 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@Service
@TimescaleDBTsDao
@PsqlDao
@Profile("install")
@Slf4j
public class TimescaleTsDatabaseSchemaService extends SqlAbstractDatabaseSchemaService implements TsDatabaseSchemaService {

2
application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java

@ -22,7 +22,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.util.PsqlDao;
import org.thingsboard.server.dao.util.TimescaleDBTsDao;
import java.io.File;
@ -37,7 +36,6 @@ import java.sql.DriverManager;
@Profile("install")
@Slf4j
@TimescaleDBTsDao
@PsqlDao
public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeService implements DatabaseTsUpgradeService {
@Value("${sql.timescale.chunk_time_interval:86400000}")

53
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -21,7 +21,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
@ -56,12 +55,9 @@ import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.alarm.AlarmDao;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.model.sql.DeviceProfileEntity;
import org.thingsboard.server.dao.model.sql.RelationEntity;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
@ -69,7 +65,6 @@ import org.thingsboard.server.dao.sql.device.DeviceProfileRepository;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.install.SystemDataLoaderService;
import org.thingsboard.server.service.install.TbRuleEngineQueueConfigService;
@ -107,9 +102,6 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private TimeseriesService tsService;
@Autowired
private AlarmService alarmService;
@Autowired
private EntityService entityService;
@ -120,7 +112,7 @@ public class DefaultDataUpdateService implements DataUpdateService {
private DeviceProfileRepository deviceProfileRepository;
@Autowired
private OAuth2Service oAuth2Service;
private RateLimitsUpdater rateLimitsUpdater;
@Autowired
private TenantProfileService tenantProfileService;
@ -162,23 +154,9 @@ public class DefaultDataUpdateService implements DataUpdateService {
break;
case "3.3.4":
log.info("Updating data from version 3.3.4 to 3.4.0 ...");
log.info("Loading queues...");
try {
if (!CollectionUtils.isEmpty(queueConfig.getQueues())) {
queueConfig.getQueues().forEach(queueSettings -> {
Queue queue = queueConfigToQueue(queueSettings);
Queue existing = queueService.findQueueByTenantIdAndName(queue.getTenantId(), queue.getName());
if (existing == null) {
queueService.saveQueue(queue);
}
});
} else {
systemDataLoaderService.createQueues();
}
} catch (Exception e) {
}
tenantsProfileQueueConfigurationUpdater.updateEntities(null);
checkPointRuleNodesUpdater.updateEntities(null);
rateLimitsUpdater.updateEntities();
tenantsProfileQueueConfigurationUpdater.updateEntities();
checkPointRuleNodesUpdater.updateEntities();
break;
default:
throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion);
@ -649,29 +627,6 @@ public class DefaultDataUpdateService implements DataUpdateService {
return mainQueueConfiguration;
}
private Queue queueConfigToQueue(TbRuleEngineQueueConfiguration queueSettings) {
Queue queue = new Queue();
queue.setTenantId(TenantId.SYS_TENANT_ID);
queue.setName(queueSettings.getName());
queue.setTopic(queueSettings.getTopic());
queue.setPollInterval(queueSettings.getPollInterval());
queue.setPartitions(queueSettings.getPartitions());
queue.setPackProcessingTimeout(queueSettings.getPackProcessingTimeout());
SubmitStrategy submitStrategy = new SubmitStrategy();
submitStrategy.setBatchSize(queueSettings.getSubmitStrategy().getBatchSize());
submitStrategy.setType(SubmitStrategyType.valueOf(queueSettings.getSubmitStrategy().getType()));
queue.setSubmitStrategy(submitStrategy);
ProcessingStrategy processingStrategy = new ProcessingStrategy();
processingStrategy.setType(ProcessingStrategyType.valueOf(queueSettings.getProcessingStrategy().getType()));
processingStrategy.setRetries(queueSettings.getProcessingStrategy().getRetries());
processingStrategy.setFailurePercentage(queueSettings.getProcessingStrategy().getFailurePercentage());
processingStrategy.setPauseBetweenRetries(queueSettings.getProcessingStrategy().getPauseBetweenRetries());
processingStrategy.setMaxPauseBetweenRetries(queueSettings.getProcessingStrategy().getMaxPauseBetweenRetries());
queue.setProcessingStrategy(processingStrategy);
queue.setConsumerPerPartition(queueSettings.isConsumerPerPartition());
return queue;
}
private final PaginatedUpdater<String, RuleNode> checkPointRuleNodesUpdater =
new PaginatedUpdater<>() {

4
application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java

@ -49,6 +49,10 @@ public abstract class PaginatedUpdater<I, D> {
}
}
public void updateEntities() {
updateEntities(null);
}
protected boolean forceReportTotal() {
return false;
}

115
application/src/main/java/org/thingsboard/server/service/install/update/RateLimitsUpdater.java

@ -0,0 +1,115 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install.update;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.tenant.TenantProfileService;
@Component
class RateLimitsUpdater extends PaginatedUpdater<String, TenantProfile> {
@Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_TENANT_ENABLED') ?: environment.getProperty('server.rest.limits.tenant.enabled') ?: 'false' }")
boolean tenantServerRestLimitsEnabled;
@Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_TENANT_CONFIGURATION') ?: environment.getProperty('server.rest.limits.tenant.configuration') ?: '100:1,2000:60' }")
String tenantServerRestLimitsConfiguration;
@Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_CUSTOMER_ENABLED') ?: environment.getProperty('server.rest.limits.customer.enabled') ?: 'false' }")
boolean customerServerRestLimitsEnabled;
@Value("#{ environment.getProperty('TB_SERVER_REST_LIMITS_CUSTOMER_CONFIGURATION') ?: environment.getProperty('server.rest.limits.customer.configuration') ?: '50:1,1000:60' }")
String customerServerRestLimitsConfiguration;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_TENANT') ?: environment.getProperty('server.ws.limits.max_sessions_per_tenant') ?: '0' }")
private int maxWsSessionsPerTenant;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_CUSTOMER') ?: environment.getProperty('server.ws.limits.max_sessions_per_customer') ?: '0' }")
private int maxWsSessionsPerCustomer;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_REGULAR_USER') ?: environment.getProperty('server.ws.limits.max_sessions_per_regular_user') ?: '0' }")
private int maxWsSessionsPerRegularUser;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SESSIONS_PER_PUBLIC_USER') ?: environment.getProperty('server.ws.limits.max_sessions_per_public_user') ?: '0' }")
private int maxWsSessionsPerPublicUser;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_QUEUE_PER_WS_SESSION') ?: environment.getProperty('server.ws.limits.max_queue_per_ws_session') ?: '500' }")
private int wsMsgQueueLimitPerSession;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_TENANT') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_tenant') ?: '0' }")
private long maxWsSubscriptionsPerTenant;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_CUSTOMER') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_customer') ?: '0' }")
private long maxWsSubscriptionsPerCustomer;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_REGULAR_USER') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_regular_user') ?: '0' }")
private long maxWsSubscriptionsPerRegularUser;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_SUBSCRIPTIONS_PER_PUBLIC_USER') ?: environment.getProperty('server.ws.limits.max_subscriptions_per_public_user') ?: '0' }")
private long maxWsSubscriptionsPerPublicUser;
@Value("#{ environment.getProperty('TB_SERVER_WS_TENANT_RATE_LIMITS_MAX_UPDATES_PER_SESSION') ?: environment.getProperty('server.ws.limits.max_updates_per_session') ?: '300:1,3000:60' }")
private String wsUpdatesPerSessionRateLimit;
@Value("#{ environment.getProperty('CASSANDRA_QUERY_TENANT_RATE_LIMITS_ENABLED') ?: environment.getProperty('cassandra.query.tenant_rate_limits.enabled') ?: 'false' }")
private boolean cassandraQueryTenantRateLimitsEnabled;
@Value("#{ environment.getProperty('CASSANDRA_QUERY_TENANT_RATE_LIMITS_CONFIGURATION') ?: environment.getProperty('cassandra.query.tenant_rate_limits.configuration') ?: '1000:1,30000:60' }")
private String cassandraQueryTenantRateLimitsConfiguration;
@Autowired
private TenantProfileService tenantProfileService;
@Override
protected boolean forceReportTotal() {
return true;
}
@Override
protected String getName() {
return "Rate limits updater";
}
@Override
protected PageData<TenantProfile> findEntities(String id, PageLink pageLink) {
return tenantProfileService.findTenantProfiles(TenantId.SYS_TENANT_ID, pageLink);
}
@Override
protected void updateEntity(TenantProfile tenantProfile) {
var profileConfiguration = tenantProfile.getDefaultProfileConfiguration();
if (tenantServerRestLimitsEnabled && StringUtils.isNotEmpty(tenantServerRestLimitsConfiguration)) {
profileConfiguration.setTenantServerRestLimitsConfiguration(tenantServerRestLimitsConfiguration);
}
if (customerServerRestLimitsEnabled && StringUtils.isNotEmpty(customerServerRestLimitsConfiguration)) {
profileConfiguration.setCustomerServerRestLimitsConfiguration(customerServerRestLimitsConfiguration);
}
profileConfiguration.setMaxWsSessionsPerTenant(maxWsSessionsPerTenant);
profileConfiguration.setMaxWsSessionsPerCustomer(maxWsSessionsPerCustomer);
profileConfiguration.setMaxWsSessionsPerPublicUser(maxWsSessionsPerPublicUser);
profileConfiguration.setMaxWsSessionsPerRegularUser(maxWsSessionsPerRegularUser);
profileConfiguration.setMaxWsSubscriptionsPerTenant(maxWsSubscriptionsPerTenant);
profileConfiguration.setMaxWsSubscriptionsPerCustomer(maxWsSubscriptionsPerCustomer);
profileConfiguration.setMaxWsSubscriptionsPerPublicUser(maxWsSubscriptionsPerPublicUser);
profileConfiguration.setMaxWsSubscriptionsPerRegularUser(maxWsSubscriptionsPerRegularUser);
profileConfiguration.setWsMsgQueueLimitPerSession(wsMsgQueueLimitPerSession);
if (StringUtils.isNotEmpty(wsUpdatesPerSessionRateLimit)) {
profileConfiguration.setWsUpdatesPerSessionRateLimit(wsUpdatesPerSessionRateLimit);
}
if (cassandraQueryTenantRateLimitsEnabled && StringUtils.isNotEmpty(cassandraQueryTenantRateLimitsConfiguration)) {
profileConfiguration.setCassandraQueryTenantRateLimitsConfiguration(cassandraQueryTenantRateLimitsConfiguration);
}
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile);
}
}

11
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -53,7 +53,7 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
@ -135,6 +135,15 @@ public class DefaultTbClusterService implements TbClusterService {
toCoreMsgs.incrementAndGet();
}
@Override
public void pushMsgToVersionControl(TenantId tenantId, TransportProtos.ToVersionControlServiceMsg msg, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_VC_EXECUTOR, tenantId, tenantId);
log.trace("PUSHING msg: {} to:{}", msg, tpi);
producerProvider.getTbVersionControlMsgProducer().send(tpi, new TbProtoQueueMsg<>(tenantId.getId(), msg), callback);
//TODO: ashvayka
toCoreMsgs.incrementAndGet();
}
@Override
public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);

13
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -36,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
@ -75,6 +75,8 @@ import org.thingsboard.server.service.state.DeviceStateService;
import org.thingsboard.server.service.subscription.SubscriptionManagerService;
import org.thingsboard.server.service.subscription.TbLocalSubscriptionService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.sync.vc.GitVersionControlQueueService;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import javax.annotation.PostConstruct;
@ -117,6 +119,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private final TbCoreDeviceRpcService tbCoreDeviceRpcService;
private final EdgeNotificationService edgeNotificationService;
private final OtaPackageStateService firmwareStateService;
private final GitVersionControlQueueService vcQueueService;
private final TbCoreConsumerStats stats;
protected final TbQueueConsumer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> usageStatsConsumer;
private final TbQueueConsumer<TbProtoQueueMsg<ToOtaPackageStateServiceMsg>> firmwareStatesConsumer;
@ -138,7 +141,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService,
EdgeNotificationService edgeNotificationService,
OtaPackageStateService firmwareStateService, PartitionService partitionService) {
OtaPackageStateService firmwareStateService,
GitVersionControlQueueService vcQueueService,
PartitionService partitionService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, partitionService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
@ -151,6 +156,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
this.stats = new TbCoreConsumerStats(statsFactory);
this.statsService = statsService;
this.firmwareStateService = firmwareStateService;
this.vcQueueService = vcQueueService;
}
@PostConstruct
@ -322,6 +328,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
TransportProtos.QueueDeleteMsg queue = toCoreNotification.getQueueDeleteMsg();
partitionService.removeQueue(queue);
callback.onSuccess();
} else if (toCoreNotification.hasVcResponseMsg()) {
vcQueueService.processResponse(toCoreNotification.getVcResponseMsg());
callback.onSuccess();
}
if (statsEnabled) {
stats.log(toCoreNotification);

3
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -36,7 +36,7 @@ import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -75,7 +75,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;

4
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -37,7 +37,7 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.util.AfterStartUp;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
@ -160,7 +160,7 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
TbActorMsg actorMsg = actorMsgOpt.get();
if (actorMsg instanceof ComponentLifecycleMsg) {
ComponentLifecycleMsg componentLifecycleMsg = (ComponentLifecycleMsg) actorMsg;
log.info("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(),
log.debug("[{}][{}][{}] Received Lifecycle event: {}", componentLifecycleMsg.getTenantId(), componentLifecycleMsg.getEntityId().getEntityType(),
componentLifecycleMsg.getEntityId(), componentLifecycleMsg.getEvent());
if (EntityType.TENANT_PROFILE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
TenantProfileId tenantProfileId = new TenantProfileId(componentLifecycleMsg.getEntityId().getId());

24
application/src/main/java/org/thingsboard/server/service/rule/DefaultTbRuleChainService.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.rule;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.flow.TbRuleChainInputNode;
@ -46,15 +47,9 @@ import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.*;
import java.util.stream.Collectors;
@RequiredArgsConstructor
@ -66,6 +61,8 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement
private final RuleChainService ruleChainService;
private final RelationService relationService;
private final EntitiesVersionControlService vcService;
@Override
public Set<String> getRuleChainOutputLabels(TenantId tenantId, RuleChainId ruleChainId) {
RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId);
@ -169,6 +166,7 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement
ActionType actionType = ruleChain.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
try {
RuleChain savedRuleChain = checkNotNull(ruleChainService.saveRuleChain(ruleChain));
vcService.autoCommit(user, savedRuleChain.getId());
if (RuleChainType.CORE.equals(savedRuleChain.getType())) {
tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedRuleChain.getId(),
@ -222,6 +220,7 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement
public RuleChain saveDefaultByName(TenantId tenantId, DefaultRuleChainCreateRequest request, SecurityUser user) throws ThingsboardException {
try {
RuleChain savedRuleChain = installScripts.createDefaultRuleChain(tenantId, request.getName());
vcService.autoCommit(user, savedRuleChain.getId());
tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedRuleChain.getId(), ComponentLifecycleEvent.CREATED);
notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedRuleChain.getId(),
savedRuleChain, user, ActionType.ADDED, false, null);
@ -281,6 +280,15 @@ public class DefaultTbRuleChainService extends AbstractTbEntityService implement
updatedRuleChains = Collections.emptyList();
}
if (updatedRuleChains.isEmpty()) {
vcService.autoCommit(user, ruleChainMetaData.getRuleChainId());
} else {
List<UUID> uuids = new ArrayList<>(updatedRuleChains.size() + 1);
uuids.add(ruleChainMetaData.getRuleChainId().getId());
updatedRuleChains.forEach(rc -> uuids.add(rc.getId().getId()));
vcService.autoCommit(user, EntityType.RULE_CHAIN, uuids);
}
RuleChainMetaData savedRuleChainMetaData = checkNotNull(ruleChainService.loadRuleChainMetaData(tenantId, ruleChainMetaDataId));
if (RuleChainType.CORE.equals(ruleChain.getType())) {

2
application/src/main/java/org/thingsboard/server/service/security/auth/mfa/DefaultTwoFactorAuthService.java

@ -149,7 +149,7 @@ public class DefaultTwoFactorAuthService implements TwoFactorAuthService {
ConcurrentMap<TwoFaProviderType, TbRateLimits> providersRateLimits = rateLimits.computeIfAbsent(userId, i -> new ConcurrentHashMap<>());
TbRateLimits rateLimit = providersRateLimits.get(providerType);
if (rateLimit == null || !rateLimit.getConfig().equals(rateLimitConfig)) {
if (rateLimit == null || !rateLimit.getConfiguration().equals(rateLimitConfig)) {
rateLimit = new TbRateLimits(rateLimitConfig, true);
providersRateLimits.put(providerType, rateLimit);
}

27
application/src/main/java/org/thingsboard/server/service/security/auth/mfa/config/DefaultTwoFaConfigManager.java

@ -56,12 +56,31 @@ public class DefaultTwoFaConfigManager implements TwoFaConfigManager {
@Override
public Optional<AccountTwoFaSettings> getAccountTwoFaSettings(TenantId tenantId, UserId userId) {
PlatformTwoFaSettings platformTwoFaSettings = getPlatformTwoFaSettings(tenantId, true).orElse(null);
return Optional.ofNullable(userAuthSettingsDao.findByUserId(userId))
.flatMap(userAuthSettings -> Optional.ofNullable(userAuthSettings.getTwoFaSettings()))
.map(twoFaSettings -> {
twoFaSettings.getConfigs().keySet().removeIf(providerType -> {
return getTwoFaProviderConfig(tenantId, providerType).isEmpty();
.map(userAuthSettings -> {
AccountTwoFaSettings twoFaSettings = userAuthSettings.getTwoFaSettings();
if (twoFaSettings == null) return null;
boolean updateNeeded;
Map<TwoFaProviderType, TwoFaAccountConfig> configs = twoFaSettings.getConfigs();
updateNeeded = configs.keySet().removeIf(providerType -> {
return platformTwoFaSettings == null || platformTwoFaSettings.getProviderConfig(providerType).isEmpty();
});
if (configs.size() == 1 && configs.containsKey(TwoFaProviderType.BACKUP_CODE)) {
configs.remove(TwoFaProviderType.BACKUP_CODE);
updateNeeded = true;
}
if (!configs.isEmpty() && configs.values().stream().noneMatch(TwoFaAccountConfig::isUseByDefault)) {
configs.values().stream()
.filter(config -> config.getProviderType() != TwoFaProviderType.BACKUP_CODE)
.findFirst().ifPresent(config -> config.setUseByDefault(true));
updateNeeded = true;
}
if (updateNeeded) {
twoFaSettings = saveAccountTwoFaSettings(tenantId, userId, twoFaSettings);
}
return twoFaSettings;
});
}

2
application/src/main/java/org/thingsboard/server/service/security/auth/mfa/provider/impl/SmsTwoFaProvider.java

@ -63,7 +63,7 @@ public class SmsTwoFaProvider extends OtpBasedTwoFaProvider<SmsTwoFaProviderConf
@Override
public void check(TenantId tenantId) throws ThingsboardException {
if (!smsService.isConfigured(tenantId)) {
throw new ThingsboardException("SMS service in not configured", ThingsboardErrorCode.BAD_REQUEST_PARAMS);
throw new ThingsboardException("SMS service is not configured", ThingsboardErrorCode.BAD_REQUEST_PARAMS);
}
}

3
application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@ -41,7 +41,8 @@ public enum Resource {
OTA_PACKAGE(EntityType.OTA_PACKAGE),
EDGE(EntityType.EDGE),
RPC(EntityType.RPC),
QUEUE(EntityType.QUEUE);
QUEUE(EntityType.QUEUE),
VERSION_CONTROL;
private final EntityType entityType;

2
application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java

@ -28,6 +28,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
public TenantAdminPermissions() {
super();
put(Resource.ADMIN_SETTINGS, PermissionChecker.allowAllPermissionChecker);
put(Resource.ALARM, tenantEntityPermissionChecker);
put(Resource.ASSET, tenantEntityPermissionChecker);
put(Resource.DEVICE, tenantEntityPermissionChecker);
@ -46,6 +47,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
put(Resource.EDGE, tenantEntityPermissionChecker);
put(Resource.RPC, tenantEntityPermissionChecker);
put(Resource.QUEUE, queuePermissionChecker);
put(Resource.VERSION_CONTROL, PermissionChecker.allowAllPermissionChecker);
}
public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {

1
application/src/main/java/org/thingsboard/server/service/security/system/DefaultSystemSecurityService.java

@ -119,6 +119,7 @@ public class DefaultSystemSecurityService implements SystemSecurityService {
AdminSettings adminSettings = adminSettingsService.findAdminSettingsByKey(tenantId, "securitySettings");
if (adminSettings == null) {
adminSettings = new AdminSettings();
adminSettings.setTenantId(tenantId);
adminSettings.setKey("securitySettings");
}
adminSettings.setJsonValue(JacksonUtil.valueToTree(securitySettings));

2
application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java

@ -33,7 +33,7 @@ import org.thingsboard.server.gen.transport.TransportProtos;
public class SessionRedisCache extends RedisTbTransactionalCache<DeviceId, TransportProtos.DeviceSessionsCacheEntry> {
public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() {
super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() {
@Override
public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException {
return deviceSessionsCacheEntry.toByteArray();

166
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -0,0 +1,166 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.apiusage.RateLimitService;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService;
import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
@TbCoreComponent
@RequiredArgsConstructor
@Slf4j
public class DefaultEntitiesExportImportService implements EntitiesExportImportService {
private final Map<EntityType, EntityExportService<?, ?, ?>> exportServices = new HashMap<>();
private final Map<EntityType, EntityImportService<?, ?, ?>> importServices = new HashMap<>();
private final EntityActionService entityActionService;
private final RelationService relationService;
private final RateLimitService rateLimitService;
protected static final List<EntityType> SUPPORTED_ENTITY_TYPES = List.of(
EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN,
EntityType.DASHBOARD, EntityType.DEVICE_PROFILE, EntityType.DEVICE,
EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE
);
@Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {
if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) {
throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS);
}
EntityType entityType = entityId.getEntityType();
EntityExportService<I, E, EntityExportData<E>> exportService = getExportService(entityType);
return exportService.getExportData(ctx, entityId);
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(EntitiesImportCtx ctx, EntityExportData<E> exportData) throws ThingsboardException {
if (!rateLimitService.checkEntityImportLimit(ctx.getTenantId())) {
throw new ThingsboardException("Rate limit for entities import is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS);
}
if (exportData.getEntity() == null || exportData.getEntity().getId() == null) {
throw new DataValidationException("Invalid entity data");
}
EntityType entityType = exportData.getEntityType();
EntityImportService<I, E, EntityExportData<E>> importService = getImportService(entityType);
EntityImportResult<E> importResult = importService.importEntity(ctx, exportData);
ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId());
ctx.addReferenceCallback(importResult.getSaveReferencesCallback());
ctx.addEventCallback(importResult.getSendEventsCallback());
return importResult;
}
@Override
public void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException {
for (ThrowingRunnable saveReferencesCallback : ctx.getReferenceCallbacks()) {
saveReferencesCallback.run();
}
relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations()));
for (EntityRelation relation : ctx.getRelations()) {
entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null,
ActionType.RELATION_ADD_OR_UPDATE, null, relation);
entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null,
ActionType.RELATION_ADD_OR_UPDATE, null, relation);
}
}
@Override
public Comparator<EntityType> getEntityTypeComparatorForImport() {
return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf);
}
@SuppressWarnings("unchecked")
private <I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> EntityExportService<I, E, D> getExportService(EntityType entityType) {
EntityExportService<?, ?, ?> exportService = exportServices.get(entityType);
if (exportService == null) {
throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported");
}
return (EntityExportService<I, E, D>) exportService;
}
@SuppressWarnings("unchecked")
private <I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> EntityImportService<I, E, D> getImportService(EntityType entityType) {
EntityImportService<?, ?, ?> importService = importServices.get(entityType);
if (importService == null) {
throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported");
}
return (EntityImportService<I, E, D>) importService;
}
@Autowired
private void setExportServices(DefaultEntityExportService<?, ?, ?> defaultExportService,
Collection<BaseEntityExportService<?, ?, ?>> exportServices) {
exportServices.stream()
.sorted(Comparator.comparing(exportService -> exportService.getSupportedEntityTypes().size(), Comparator.reverseOrder()))
.forEach(exportService -> {
exportService.getSupportedEntityTypes().forEach(entityType -> {
this.exportServices.put(entityType, exportService);
});
});
SUPPORTED_ENTITY_TYPES.forEach(entityType -> {
this.exportServices.putIfAbsent(entityType, defaultExportService);
});
}
@Autowired
private void setImportServices(Collection<EntityImportService<?, ?, ?>> importServices) {
importServices.forEach(entityImportService -> {
this.importServices.put(entityImportService.getEntityType(), entityImportService);
});
}
}

40
application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java

@ -0,0 +1,40 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import java.util.Comparator;
public interface EntitiesExportImportService {
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> exportEntity(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException;
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> importEntity(EntitiesImportCtx ctx, EntityExportData<E> exportData) throws ThingsboardException;
void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException;
Comparator<EntityType> getEntityTypeComparatorForImport();
}

210
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java

@ -0,0 +1,210 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.ExportableEntityDao;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.permission.AccessControlService;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.function.BiConsumer;
@Service
@TbCoreComponent
@RequiredArgsConstructor
@Slf4j
public class DefaultExportableEntitiesService implements ExportableEntitiesService {
private final Map<EntityType, Dao<?>> daos = new HashMap<>();
private final Map<EntityType, BiConsumer<TenantId, EntityId>> removers = new HashMap<>();
private final AccessControlService accessControlService;
@Override
public <E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId) {
EntityType entityType = externalId.getEntityType();
Dao<E> dao = getDao(entityType);
E entity = null;
if (dao instanceof ExportableEntityDao) {
ExportableEntityDao<I, E> exportableEntityDao = (ExportableEntityDao<I, E>) dao;
entity = exportableEntityDao.findByTenantIdAndExternalId(tenantId.getId(), externalId.getId());
}
if (entity == null || !belongsToTenant(entity, tenantId)) {
return null;
}
return entity;
}
@Override
public <E extends HasId<I>, I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id) {
E entity = findEntityById(id);
if (entity == null || !belongsToTenant(entity, tenantId)) {
return null;
}
return entity;
}
@Override
public <E extends HasId<I>, I extends EntityId> E findEntityById(I id) {
EntityType entityType = id.getEntityType();
Dao<E> dao = getDao(entityType);
if (dao == null) {
throw new IllegalArgumentException("Unsupported entity type " + entityType);
}
return dao.findById(TenantId.SYS_TENANT_ID, id.getId());
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name) {
Dao<E> dao = getDao(entityType);
E entity = null;
if (dao instanceof ExportableEntityDao) {
ExportableEntityDao<I, E> exportableEntityDao = (ExportableEntityDao<I, E>) dao;
try {
entity = exportableEntityDao.findByTenantIdAndName(tenantId.getId(), name);
} catch (UnsupportedOperationException ignored) {
}
}
if (entity == null || !belongsToTenant(entity, tenantId)) {
return null;
}
return entity;
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink) {
ExportableEntityDao<I, E> dao = getExportableEntityDao(entityType);
if (dao != null) {
return dao.findByTenantId(tenantId.getId(), pageLink);
} else {
return new PageData<>();
}
}
@Override
public <I extends EntityId> I getExternalIdByInternal(I internalId) {
ExportableEntityDao<I, ?> dao = getExportableEntityDao(internalId.getEntityType());
if (dao != null) {
return dao.getExternalIdByInternal(internalId);
} else {
return null;
}
}
private boolean belongsToTenant(HasId<? extends EntityId> entity, TenantId tenantId) {
return tenantId.equals(((HasTenantId) entity).getTenantId());
}
@Override
public <I extends EntityId> void removeById(TenantId tenantId, I id) {
EntityType entityType = id.getEntityType();
BiConsumer<TenantId, EntityId> entityRemover = removers.get(entityType);
if (entityRemover == null) {
throw new IllegalArgumentException("Unsupported entity type " + entityType);
}
entityRemover.accept(tenantId, id);
}
private <I extends EntityId, E extends ExportableEntity<I>> ExportableEntityDao<I, E> getExportableEntityDao(EntityType entityType) {
Dao<E> dao = getDao(entityType);
if (dao instanceof ExportableEntityDao) {
return (ExportableEntityDao<I, E>) dao;
} else {
return null;
}
}
@SuppressWarnings("unchecked")
private <E> Dao<E> getDao(EntityType entityType) {
return (Dao<E>) daos.get(entityType);
}
@Autowired
private void setDaos(Collection<Dao<?>> daos) {
daos.forEach(dao -> {
if (dao.getEntityType() != null) {
this.daos.put(dao.getEntityType(), dao);
}
});
}
@Autowired
private void setRemovers(CustomerService customerService, AssetService assetService, RuleChainService ruleChainService,
DashboardService dashboardService, DeviceProfileService deviceProfileService,
DeviceService deviceService, WidgetsBundleService widgetsBundleService) {
removers.put(EntityType.CUSTOMER, (tenantId, entityId) -> {
customerService.deleteCustomer(tenantId, (CustomerId) entityId);
});
removers.put(EntityType.ASSET, (tenantId, entityId) -> {
assetService.deleteAsset(tenantId, (AssetId) entityId);
});
removers.put(EntityType.RULE_CHAIN, (tenantId, entityId) -> {
ruleChainService.deleteRuleChainById(tenantId, (RuleChainId) entityId);
});
removers.put(EntityType.DASHBOARD, (tenantId, entityId) -> {
dashboardService.deleteDashboard(tenantId, (DashboardId) entityId);
});
removers.put(EntityType.DEVICE_PROFILE, (tenantId, entityId) -> {
deviceProfileService.deleteDeviceProfile(tenantId, (DeviceProfileId) entityId);
});
removers.put(EntityType.DEVICE, (tenantId, entityId) -> {
deviceService.deleteDevice(tenantId, (DeviceId) entityId);
});
removers.put(EntityType.WIDGETS_BUNDLE, (tenantId, entityId) -> {
widgetsBundleService.deleteWidgetsBundle(tenantId, (WidgetsBundleId) entityId);
});
}
}

28
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java

@ -0,0 +1,28 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
public interface EntityExportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> {
D getExportData(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException;
}

42
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
public interface ExportableEntitiesService {
<E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndExternalId(TenantId tenantId, I externalId);
<E extends HasId<I>, I extends EntityId> E findEntityByTenantIdAndId(TenantId tenantId, I id);
<E extends HasId<I>, I extends EntityId> E findEntityById(I id);
<E extends ExportableEntity<I>, I extends EntityId> E findEntityByTenantIdAndName(TenantId tenantId, EntityType entityType, String name);
<E extends ExportableEntity<I>, I extends EntityId> PageData<E> findEntitiesByTenantId(TenantId tenantId, EntityType entityType, PageLink pageLink);
<I extends EntityId> I getExternalIdByInternal(I internalId);
<I extends EntityId> void removeById(TenantId tenantId, I id);
}

42
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/AssetExportService.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
@Service
@TbCoreComponent
public class AssetExportService extends BaseEntityExportService<AssetId, Asset, EntityExportData<Asset>> {
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Asset asset, EntityExportData<Asset> exportData) {
asset.setCustomerId(getExternalIdOrElseInternal(ctx, asset.getCustomerId()));
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.ASSET);
}
}

52
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java

@ -0,0 +1,52 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import com.fasterxml.jackson.databind.JsonNode;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
public abstract class BaseEntityExportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> extends DefaultEntityExportService<I, E, D> {
@Override
protected void setAdditionalExportData(EntitiesExportCtx<?> ctx, E entity, D exportData) throws ThingsboardException {
setRelatedEntities(ctx, entity, (D) exportData);
super.setAdditionalExportData(ctx, entity, exportData);
}
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, E mainEntity, D exportData) {
}
protected D newExportData() {
return (D) new EntityExportData<E>();
}
;
public abstract Set<EntityType> getSupportedEntityTypes();
protected void replaceUuidsRecursively(EntitiesExportCtx<?> ctx, JsonNode node, Set<String> skipFieldsSet) {
JacksonUtil.replaceUuidsRecursively(node, skipFieldsSet, uuid -> getExternalIdOrElseInternalByUuid(ctx, uuid));
}
}

61
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java

@ -0,0 +1,61 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.common.util.RegexUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Set;
import java.util.UUID;
@Service
@TbCoreComponent
public class DashboardExportService extends BaseEntityExportService<DashboardId, Dashboard, EntityExportData<Dashboard>> {
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData) {
if (CollectionUtils.isNotEmpty(dashboard.getAssignedCustomers())) {
dashboard.getAssignedCustomers().forEach(customerInfo -> {
customerInfo.setCustomerId(getExternalIdOrElseInternal(ctx, customerInfo.getCustomerId()));
});
}
for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) {
replaceUuidsRecursively(ctx, entityAlias, Collections.emptySet());
}
for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) {
replaceUuidsRecursively(ctx, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id"));
}
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.DASHBOARD);
}
}

184
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java

@ -0,0 +1,184 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Service
@TbCoreComponent
@Primary
public class DefaultEntityExportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> implements EntityExportService<I, E, D> {
@Autowired
@Lazy
protected ExportableEntitiesService exportableEntitiesService;
@Autowired
private RelationService relationService;
@Autowired
private AttributesService attributesService;
@Override
public final D getExportData(EntitiesExportCtx<?> ctx, I entityId) throws ThingsboardException {
D exportData = newExportData();
E entity = exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entityId);
if (entity == null) {
throw new IllegalArgumentException(entityId.getEntityType() + " [" + entityId.getId() + "] not found");
}
exportData.setEntity(entity);
exportData.setEntityType(entityId.getEntityType());
setAdditionalExportData(ctx, entity, exportData);
var externalId = entity.getExternalId() != null ? entity.getExternalId() : entity.getId();
ctx.putExternalId(entityId, externalId);
entity.setId(externalId);
entity.setTenantId(null);
return exportData;
}
protected void setAdditionalExportData(EntitiesExportCtx<?> ctx, E entity, D exportData) throws ThingsboardException {
var exportSettings = ctx.getSettings();
if (exportSettings.isExportRelations()) {
List<EntityRelation> relations = exportRelations(ctx, entity);
relations.forEach(relation -> {
relation.setFrom(getExternalIdOrElseInternal(ctx, relation.getFrom()));
relation.setTo(getExternalIdOrElseInternal(ctx, relation.getTo()));
});
exportData.setRelations(relations);
}
if (exportSettings.isExportAttributes()) {
Map<String, List<AttributeExportData>> attributes = exportAttributes(ctx, entity);
exportData.setAttributes(attributes);
}
}
private List<EntityRelation> exportRelations(EntitiesExportCtx<?> ctx, E entity) throws ThingsboardException {
List<EntityRelation> relations = new ArrayList<>();
List<EntityRelation> inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
relations.addAll(inboundRelations);
List<EntityRelation> outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
relations.addAll(outboundRelations);
return relations;
}
private Map<String, List<AttributeExportData>> exportAttributes(EntitiesExportCtx<?> ctx, E entity) throws ThingsboardException {
List<String> scopes;
if (entity.getId().getEntityType() == EntityType.DEVICE) {
scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE);
} else {
scopes = Collections.singletonList(DataConstants.SERVER_SCOPE);
}
Map<String, List<AttributeExportData>> attributes = new LinkedHashMap<>();
scopes.forEach(scope -> {
try {
attributes.put(scope, attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream()
.map(attribute -> {
AttributeExportData attributeExportData = new AttributeExportData();
attributeExportData.setKey(attribute.getKey());
attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs());
attributeExportData.setStrValue(attribute.getStrValue().orElse(null));
attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null));
attributeExportData.setLongValue(attribute.getLongValue().orElse(null));
attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null));
attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null));
return attributeExportData;
})
.collect(Collectors.toList()));
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
});
return attributes;
}
protected <ID extends EntityId> ID getExternalIdOrElseInternal(EntitiesExportCtx<?> ctx, ID internalId) {
if (internalId == null || internalId.isNullUid()) return internalId;
var result = ctx.getExternalId(internalId);
if (result == null) {
result = Optional.ofNullable(exportableEntitiesService.getExternalIdByInternal(internalId))
.orElse(internalId);
ctx.putExternalId(internalId, result);
}
return result;
}
protected UUID getExternalIdOrElseInternalByUuid(EntitiesExportCtx<?> ctx, UUID internalUuid) {
for (EntityType entityType : EntityType.values()) {
EntityId internalId;
try {
internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid);
} catch (Exception e) {
continue;
}
EntityId externalId = ctx.getExternalId(internalId);
if (externalId != null) {
return externalId.getId();
}
}
for (EntityType entityType : EntityType.values()) {
EntityId internalId;
try {
internalId = EntityIdFactory.getByTypeAndUuid(entityType, internalUuid);
} catch (Exception e) {
continue;
}
EntityId externalId = exportableEntitiesService.getExternalIdByInternal(internalId);
if (externalId != null) {
ctx.putExternalId(internalId, externalId);
return externalId.getId();
}
}
return internalUuid;
}
protected D newExportData() {
return (D) new EntityExportData<E>();
}
}

59
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java

@ -0,0 +1,59 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.sync.ie.DeviceExportData;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class DeviceExportService extends BaseEntityExportService<DeviceId, Device, DeviceExportData> {
private final DeviceCredentialsService deviceCredentialsService;
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, Device device, DeviceExportData exportData) {
device.setCustomerId(getExternalIdOrElseInternal(ctx, device.getCustomerId()));
device.setDeviceProfileId(getExternalIdOrElseInternal(ctx, device.getDeviceProfileId()));
if (ctx.getSettings().isExportCredentials()) {
var credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), device.getId());
credentials.setId(null);
credentials.setDeviceId(null);
exportData.setCredentials(credentials);
}
}
@Override
protected DeviceExportData newExportData() {
return new DeviceExportData();
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.DEVICE);
}
}

43
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceProfileExportService.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
@Service
@TbCoreComponent
public class DeviceProfileExportService extends BaseEntityExportService<DeviceProfileId, DeviceProfile, EntityExportData<DeviceProfile>> {
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData) {
deviceProfile.setDefaultDashboardId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultDashboardId()));
deviceProfile.setDefaultRuleChainId(getExternalIdOrElseInternal(ctx, deviceProfile.getDefaultRuleChainId()));
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.DEVICE_PROFILE);
}
}

43
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/EntityViewExportService.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
@Service
@TbCoreComponent
public class EntityViewExportService extends BaseEntityExportService<EntityViewId, EntityView, EntityExportData<EntityView>> {
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, EntityView entityView, EntityExportData<EntityView> exportData) {
entityView.setEntityId(getExternalIdOrElseInternal(ctx, entityView.getEntityId()));
entityView.setCustomerId(getExternalIdOrElseInternal(ctx, entityView.getCustomerId()));
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.ENTITY_VIEW);
}
}

76
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java

@ -0,0 +1,76 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.common.util.RegexUtils;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class RuleChainExportService extends BaseEntityExportService<RuleChainId, RuleChain, RuleChainExportData> {
private final RuleChainService ruleChainService;
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, RuleChain ruleChain, RuleChainExportData exportData) {
RuleChainMetaData metaData = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), ruleChain.getId());
Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList())
.forEach(ruleNode -> {
ruleNode.setRuleChainId(null);
ctx.putExternalId(ruleNode.getId(), ruleNode.getExternalId());
ruleNode.setId(ctx.getExternalId(ruleNode.getId()));
ruleNode.setCreatedTime(0);
ruleNode.setExternalId(null);
replaceUuidsRecursively(ctx, ruleNode.getConfiguration(), Collections.emptySet());
});
Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList())
.forEach(ruleChainConnectionInfo -> {
ruleChainConnectionInfo.setTargetRuleChainId(getExternalIdOrElseInternal(ctx, ruleChainConnectionInfo.getTargetRuleChainId()));
});
exportData.setMetaData(metaData);
if (ruleChain.getFirstRuleNodeId() != null) {
ruleChain.setFirstRuleNodeId(ctx.getExternalId(ruleChain.getFirstRuleNodeId()));
}
}
@Override
protected RuleChainExportData newExportData() {
return new RuleChainExportData();
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.RULE_CHAIN);
}
}

59
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetsBundleExportService.java

@ -0,0 +1,59 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.exporting.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.List;
import java.util.Set;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class WidgetsBundleExportService extends BaseEntityExportService<WidgetsBundleId, WidgetsBundle, WidgetsBundleExportData> {
private final WidgetTypeService widgetTypeService;
@Override
protected void setRelatedEntities(EntitiesExportCtx<?> ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData) {
if (widgetsBundle.getTenantId() == null || widgetsBundle.getTenantId().isNullUid()) {
throw new IllegalArgumentException("Export of system Widget Bundles is not allowed");
}
List<WidgetTypeDetails> widgets = widgetTypeService.findWidgetTypesDetailsByTenantIdAndBundleAlias(ctx.getTenantId(), widgetsBundle.getAlias());
exportData.setWidgets(widgets);
}
@Override
protected WidgetsBundleExportData newExportData() {
return new WidgetsBundleExportData();
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.WIDGETS_BUNDLE);
}
}

32
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
public interface EntityImportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> {
EntityImportResult<E> importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException;
EntityType getEntityType();
}

10
application/src/main/java/org/thingsboard/server/service/importing/AbstractBulkImportService.java → application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.importing;
package org.thingsboard.server.service.sync.ie.importing.csv;
import com.google.common.util.concurrent.FutureCallback;
import com.google.gson.JsonObject;
@ -44,7 +44,6 @@ import org.thingsboard.server.common.transport.adaptor.JsonConverter;
import org.thingsboard.server.controller.BaseController;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.importing.BulkImportRequest.ColumnMapping;
import org.thingsboard.server.service.security.AccessValidator;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.AccessControlService;
@ -67,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -165,7 +163,7 @@ public abstract class AbstractBulkImportService<E extends HasId<? extends Entity
protected abstract EntityType getEntityType();
private void saveKvs(SecurityUser user, E entity, Map<ColumnMapping, ParsedValue> data) {
private void saveKvs(SecurityUser user, E entity, Map<BulkImportRequest.ColumnMapping, ParsedValue> data) {
Arrays.stream(BulkImportColumnType.values())
.filter(BulkImportColumnType::isKv)
.map(kvType -> {
@ -249,7 +247,7 @@ public abstract class AbstractBulkImportService<E extends HasId<? extends Entity
linesCounter.incrementAndGet();
}
List<ColumnMapping> columnsMappings = request.getMapping().getColumns();
List<BulkImportRequest.ColumnMapping> columnsMappings = request.getMapping().getColumns();
return records.stream()
.map(record -> {
EntityData entityData = new EntityData();
@ -280,7 +278,7 @@ public abstract class AbstractBulkImportService<E extends HasId<? extends Entity
@Data
protected static class EntityData {
private final Map<BulkImportColumnType, String> fields = new LinkedHashMap<>();
private final Map<ColumnMapping, ParsedValue> kvs = new LinkedHashMap<>();
private final Map<BulkImportRequest.ColumnMapping, ParsedValue> kvs = new LinkedHashMap<>();
private int lineNumber;
}

2
application/src/main/java/org/thingsboard/server/service/importing/BulkImportColumnType.java → application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.importing;
package org.thingsboard.server.service.sync.ie.importing.csv;
import lombok.Getter;
import org.thingsboard.server.common.data.DataConstants;

2
application/src/main/java/org/thingsboard/server/service/importing/BulkImportRequest.java → application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.importing;
package org.thingsboard.server.service.sync.ie.importing.csv;
import lombok.Data;

2
application/src/main/java/org/thingsboard/server/service/importing/BulkImportResult.java → application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.importing;
package org.thingsboard.server.service.sync.ie.importing.csv;
import lombok.Data;

2
application/src/main/java/org/thingsboard/server/service/importing/ImportedEntityInfo.java → application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java

@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.importing;
package org.thingsboard.server.service.sync.ie.importing.csv;
import lombok.Data;

81
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java

@ -0,0 +1,81 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class AssetImportService extends BaseEntityImportService<AssetId, Asset, EntityExportData<Asset>> {
private final AssetService assetService;
@Override
protected void setOwner(TenantId tenantId, Asset asset, IdProvider idProvider) {
asset.setTenantId(tenantId);
asset.setCustomerId(idProvider.getInternalId(asset.getCustomerId()));
}
@Override
protected Asset prepare(EntitiesImportCtx ctx, Asset asset, Asset old, EntityExportData<Asset> exportData, IdProvider idProvider) {
return asset;
}
@Override
protected Asset saveOrUpdate(EntitiesImportCtx ctx, Asset asset, EntityExportData<Asset> exportData, IdProvider idProvider) {
return assetService.saveAsset(asset);
}
@Override
protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) throws ThingsboardException {
super.onEntitySaved(user, savedAsset, oldAsset);
if (oldAsset != null) {
entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
protected Asset deepCopy(Asset asset) {
return new Asset(asset);
}
@Override
protected void cleanupForComparison(Asset e) {
super.cleanupForComparison(e);
if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) {
e.setCustomerId(null);
}
}
@Override
public EntityType getEntityType() {
return EntityType.ASSET;
}
}

398
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -0,0 +1,398 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.api.client.util.Objects;
import com.google.common.util.concurrent.FutureCallback;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasCustomerId;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
public abstract class BaseEntityImportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> implements EntityImportService<I, E, D> {
@Autowired
@Lazy
private ExportableEntitiesService exportableEntitiesService;
@Autowired
private RelationService relationService;
@Autowired
private TelemetrySubscriptionService tsSubService;
@Autowired
protected EntityActionService entityActionService;
@Autowired
protected TbClusterService clusterService;
@Autowired
protected TbNotificationEntityService entityNotificationService;
@Transactional(rollbackFor = Exception.class)
@Override
public EntityImportResult<E> importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException {
EntityImportResult<E> importResult = new EntityImportResult<>();
ctx.setCurrentImportResult(importResult);
importResult.setEntityType(getEntityType());
IdProvider idProvider = new IdProvider(ctx, importResult);
E entity = exportData.getEntity();
entity.setExternalId(entity.getId());
E existingEntity = findExistingEntity(ctx, entity, idProvider);
importResult.setOldEntity(existingEntity);
setOwner(ctx.getTenantId(), entity, idProvider);
if (existingEntity == null) {
entity.setId(null);
} else {
entity.setId(existingEntity.getId());
entity.setCreatedTime(existingEntity.getCreatedTime());
}
E prepared = prepare(ctx, entity, existingEntity, exportData, idProvider);
boolean saveOrUpdate = existingEntity == null || compare(ctx, exportData, prepared, existingEntity);
if (saveOrUpdate) {
E savedEntity = saveOrUpdate(ctx, prepared, exportData, idProvider);
boolean created = existingEntity == null;
importResult.setCreated(created);
importResult.setUpdated(!created);
importResult.setSavedEntity(savedEntity);
ctx.putInternalId(exportData.getExternalId(), savedEntity.getId());
} else {
importResult.setSavedEntity(existingEntity);
ctx.putInternalId(exportData.getExternalId(), existingEntity.getId());
importResult.setUpdatedRelatedEntities(updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider));
}
processAfterSaved(ctx, importResult, exportData, idProvider);
return importResult;
}
protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, E prepared, D exportData, IdProvider idProvider) {
return false;
}
@Override
public abstract EntityType getEntityType();
protected abstract void setOwner(TenantId tenantId, E entity, IdProvider idProvider);
protected abstract E prepare(EntitiesImportCtx ctx, E entity, E oldEntity, D exportData, IdProvider idProvider);
protected boolean compare(EntitiesImportCtx ctx, D exportData, E prepared, E existing) {
var newCopy = deepCopy(prepared);
var existingCopy = deepCopy(existing);
cleanupForComparison(newCopy);
cleanupForComparison(existingCopy);
var result = !newCopy.equals(existingCopy);
if (result) {
log.debug("[{}] Found update.", prepared.getId());
log.debug("[{}] From: {}", prepared.getId(), newCopy);
log.debug("[{}] To: {}", prepared.getId(), existingCopy);
}
return result;
}
protected abstract E deepCopy(E e);
protected void cleanupForComparison(E e) {
e.setTenantId(null);
e.setCreatedTime(0);
}
protected abstract E saveOrUpdate(EntitiesImportCtx ctx, E entity, D exportData, IdProvider idProvider);
protected void processAfterSaved(EntitiesImportCtx ctx, EntityImportResult<E> importResult, D exportData, IdProvider idProvider) throws ThingsboardException {
E savedEntity = importResult.getSavedEntity();
E oldEntity = importResult.getOldEntity();
if (importResult.isCreated() || importResult.isUpdated()) {
importResult.addSendEventsCallback(() -> onEntitySaved(ctx.getUser(), savedEntity, oldEntity));
}
if (ctx.isUpdateRelations() && exportData.getRelations() != null) {
importRelations(ctx, exportData.getRelations(), importResult, idProvider);
}
if (ctx.isSaveAttributes() && exportData.getAttributes() != null) {
if (exportData.getAttributes().values().stream().anyMatch(d -> !d.isEmpty())) {
importResult.setUpdatedRelatedEntities(true);
}
importAttributes(ctx.getUser(), exportData.getAttributes(), importResult);
}
}
private void importRelations(EntitiesImportCtx ctx, List<EntityRelation> relations, EntityImportResult<E> importResult, IdProvider idProvider) {
var tenantId = ctx.getTenantId();
E entity = importResult.getSavedEntity();
importResult.addSaveReferencesCallback(() -> {
for (EntityRelation relation : relations) {
if (!relation.getTo().equals(entity.getId())) {
relation.setTo(idProvider.getInternalId(relation.getTo()));
}
if (!relation.getFrom().equals(entity.getId())) {
relation.setFrom(idProvider.getInternalId(relation.getFrom()));
}
}
Map<EntityRelation, EntityRelation> relationsMap = new LinkedHashMap<>();
relations.forEach(r -> relationsMap.put(r, r));
if (importResult.getOldEntity() != null) {
List<EntityRelation> existingRelations = new ArrayList<>();
existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON));
for (EntityRelation existingRelation : existingRelations) {
EntityRelation relation = relationsMap.get(existingRelation);
if (relation == null) {
importResult.setUpdatedRelatedEntities(true);
relationService.deleteRelation(tenantId, existingRelation);
importResult.addSendEventsCallback(() -> {
entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null,
ActionType.RELATION_DELETED, null, existingRelation);
entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null,
ActionType.RELATION_DELETED, null, existingRelation);
});
} else if (Objects.equal(relation.getAdditionalInfo(), existingRelation.getAdditionalInfo())) {
relationsMap.remove(relation);
}
}
}
if (!relationsMap.isEmpty()) {
importResult.setUpdatedRelatedEntities(true);
ctx.addRelations(relationsMap.values());
}
});
}
private void importAttributes(SecurityUser user, Map<String, List<AttributeExportData>> attributes, EntityImportResult<E> importResult) {
E entity = importResult.getSavedEntity();
importResult.addSaveReferencesCallback(() -> {
attributes.forEach((scope, attributesExportData) -> {
List<AttributeKvEntry> attributeKvEntries = attributesExportData.stream()
.map(attributeExportData -> {
KvEntry kvEntry;
String key = attributeExportData.getKey();
if (attributeExportData.getStrValue() != null) {
kvEntry = new StringDataEntry(key, attributeExportData.getStrValue());
} else if (attributeExportData.getBooleanValue() != null) {
kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue());
} else if (attributeExportData.getDoubleValue() != null) {
kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue());
} else if (attributeExportData.getLongValue() != null) {
kvEntry = new LongDataEntry(key, attributeExportData.getLongValue());
} else if (attributeExportData.getJsonValue() != null) {
kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue());
} else {
throw new IllegalArgumentException("Invalid attribute export data");
}
return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs());
})
.collect(Collectors.toList());
// fixme: attributes are saved outside the transaction
tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void unused) {
}
@Override
public void onFailure(Throwable thr) {
log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr);
}
});
});
});
}
protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException {
entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity,
savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(),
oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null);
}
@SuppressWarnings("unchecked")
protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
.or(() -> {
if (ctx.isFindExistingByName()) {
return Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndName(ctx.getTenantId(), getEntityType(), entity.getName()));
} else {
return Optional.empty();
}
})
.orElse(null);
}
@SuppressWarnings("unchecked")
private <ID extends EntityId> HasId<ID> findInternalEntity(TenantId tenantId, ID externalId) {
return (HasId<ID>) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId)))
.orElseThrow(() -> new MissingEntityException(externalId));
}
@SuppressWarnings("unchecked")
@RequiredArgsConstructor
protected class IdProvider {
private final EntitiesImportCtx ctx;
private final EntityImportResult<E> importResult;
public <ID extends EntityId> ID getInternalId(ID externalId) {
return getInternalId(externalId, true);
}
public <ID extends EntityId> ID getInternalId(ID externalId, boolean throwExceptionIfNotFound) {
if (externalId == null || externalId.isNullUid()) return null;
EntityId localId = ctx.getInternalId(externalId);
if (localId != null) {
return (ID) localId;
}
HasId<ID> entity;
try {
entity = findInternalEntity(ctx.getTenantId(), externalId);
} catch (Exception e) {
if (throwExceptionIfNotFound) {
throw e;
} else {
importResult.setUpdatedAllExternalIds(false);
return null;
}
}
ctx.putInternalId(externalId, entity.getId());
return entity.getId();
}
public Optional<EntityId> getInternalIdByUuid(UUID externalUuid, boolean fetchAllUUIDs, Set<EntityType> hints) {
if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty();
for (EntityType entityType : EntityType.values()) {
Optional<EntityId> externalIdOpt = buildEntityId(entityType, externalUuid);
if (!externalIdOpt.isPresent()) {
continue;
}
EntityId internalId = ctx.getInternalId(externalIdOpt.get());
if (internalId != null) {
return Optional.of(internalId);
}
}
if (fetchAllUUIDs) {
for (EntityType entityType : hints) {
Optional<EntityId> internalId = lookupInDb(externalUuid, entityType);
if (internalId.isPresent()) return internalId;
}
for (EntityType entityType : EntityType.values()) {
if (hints.contains(entityType)) {
continue;
}
Optional<EntityId> internalId = lookupInDb(externalUuid, entityType);
if (internalId.isPresent()) return internalId;
}
}
importResult.setUpdatedAllExternalIds(false);
return Optional.empty();
}
private Optional<EntityId> lookupInDb(UUID externalUuid, EntityType entityType) {
Optional<EntityId> externalIdOpt = buildEntityId(entityType, externalUuid);
if (externalIdOpt.isEmpty() || ctx.isNotFound(externalIdOpt.get())) {
return Optional.empty();
}
EntityId internalId = getInternalId(externalIdOpt.get(), false);
if (internalId != null) {
return Optional.of(internalId);
} else {
ctx.registerNotFound(externalIdOpt.get());
}
return Optional.empty();
}
private Optional<EntityId> buildEntityId(EntityType entityType, UUID externalUuid) {
try {
return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, externalUuid));
} catch (Exception e) {
return Optional.empty();
}
}
}
protected <T extends EntityId, O> T getOldEntityField(O oldEntity, Function<O, T> getter) {
return oldEntity == null ? null : getter.apply(oldEntity);
}
protected void replaceIdsRecursively(EntitiesImportCtx ctx, IdProvider idProvider, JsonNode entityAlias, Set<String> skipFieldsSet, LinkedHashSet<EntityType> hints) {
JacksonUtil.replaceUuidsRecursively(entityAlias, skipFieldsSet,
uuid -> idProvider.getInternalIdByUuid(uuid, ctx.isFinalImportAttempt(), hints).map(EntityId::getId).orElse(uuid));
}
}

84
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java

@ -0,0 +1,84 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.customer.CustomerDao;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class CustomerImportService extends BaseEntityImportService<CustomerId, Customer, EntityExportData<Customer>> {
private final CustomerService customerService;
private final CustomerDao customerDao;
@Override
protected void setOwner(TenantId tenantId, Customer customer, IdProvider idProvider) {
customer.setTenantId(tenantId);
}
@Override
protected Customer prepare(EntitiesImportCtx ctx, Customer customer, Customer old, EntityExportData<Customer> exportData, IdProvider idProvider) {
if (customer.isPublic()) {
Customer publicCustomer = customerService.findOrCreatePublicCustomer(ctx.getTenantId());
publicCustomer.setExternalId(customer.getExternalId());
return publicCustomer;
} else {
return customer;
}
}
@Override
protected Customer saveOrUpdate(EntitiesImportCtx ctx, Customer customer, EntityExportData<Customer> exportData, IdProvider idProvider) {
if (!customer.isPublic()) {
return customerService.saveCustomer(customer);
} else {
return customerDao.save(ctx.getTenantId(), customer);
}
}
@Override
protected Customer deepCopy(Customer customer) {
return new Customer(customer);
}
@Override
protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException {
super.onEntitySaved(user, savedCustomer, oldCustomer);
if (oldCustomer != null) {
entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
public EntityType getEntityType() {
return EntityType.CUSTOMER;
}
}

139
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@ -0,0 +1,139 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ShortCustomerInfo;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import org.thingsboard.common.util.RegexUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class DashboardImportService extends BaseEntityImportService<DashboardId, Dashboard, EntityExportData<Dashboard>> {
private static final LinkedHashSet<EntityType> HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.DASHBOARD, EntityType.DEVICE, EntityType.ASSET));
private final DashboardService dashboardService;
@Override
protected void setOwner(TenantId tenantId, Dashboard dashboard, IdProvider idProvider) {
dashboard.setTenantId(tenantId);
}
@Override
protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard, IdProvider idProvider) {
Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard, idProvider);
if (existingDashboard == null && ctx.isFindExistingByName()) {
existingDashboard = dashboardService.findTenantDashboardsByTitle(ctx.getTenantId(), dashboard.getName()).stream().findFirst().orElse(null);
}
return existingDashboard;
}
@Override
protected Dashboard prepare(EntitiesImportCtx ctx, Dashboard dashboard, Dashboard old, EntityExportData<Dashboard> exportData, IdProvider idProvider) {
for (JsonNode entityAlias : dashboard.getEntityAliasesConfig()) {
replaceIdsRecursively(ctx, idProvider, entityAlias, Collections.emptySet(), HINTS);
}
for (JsonNode widgetConfig : dashboard.getWidgetsConfig()) {
replaceIdsRecursively(ctx, idProvider, JacksonUtil.getSafely(widgetConfig, "config", "actions"), Collections.singleton("id"), HINTS);
}
return dashboard;
}
@Override
protected Dashboard saveOrUpdate(EntitiesImportCtx ctx, Dashboard dashboard, EntityExportData<Dashboard> exportData, IdProvider idProvider) {
var tenantId = ctx.getTenantId();
Set<ShortCustomerInfo> assignedCustomers = Optional.ofNullable(dashboard.getAssignedCustomers()).orElse(Collections.emptySet()).stream()
.peek(customerInfo -> customerInfo.setCustomerId(idProvider.getInternalId(customerInfo.getCustomerId())))
.collect(Collectors.toSet());
if (dashboard.getId() == null) {
dashboard.setAssignedCustomers(assignedCustomers);
dashboard = dashboardService.saveDashboard(dashboard);
for (ShortCustomerInfo customerInfo : assignedCustomers) {
dashboard = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerInfo.getCustomerId());
}
} else {
Set<CustomerId> existingAssignedCustomers = Optional.ofNullable(dashboardService.findDashboardById(tenantId, dashboard.getId()).getAssignedCustomers())
.orElse(Collections.emptySet()).stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet());
Set<CustomerId> newAssignedCustomers = assignedCustomers.stream().map(ShortCustomerInfo::getCustomerId).collect(Collectors.toSet());
Set<CustomerId> toUnassign = new HashSet<>(existingAssignedCustomers);
toUnassign.removeAll(newAssignedCustomers);
for (CustomerId customerId : toUnassign) {
assignedCustomers = dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers();
}
Set<CustomerId> toAssign = new HashSet<>(newAssignedCustomers);
toAssign.removeAll(existingAssignedCustomers);
for (CustomerId customerId : toAssign) {
assignedCustomers = dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId).getAssignedCustomers();
}
dashboard.setAssignedCustomers(assignedCustomers);
dashboard = dashboardService.saveDashboard(dashboard);
}
return dashboard;
}
@Override
protected Dashboard deepCopy(Dashboard dashboard) {
return new Dashboard(dashboard);
}
@Override
protected void onEntitySaved(SecurityUser user, Dashboard savedDashboard, Dashboard oldDashboard) throws ThingsboardException {
super.onEntitySaved(user, savedDashboard, oldDashboard);
if (oldDashboard != null) {
entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
public EntityType getEntityType() {
return EntityType.DASHBOARD;
}
}

105
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java

@ -0,0 +1,105 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.DeviceExportData;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class DeviceImportService extends BaseEntityImportService<DeviceId, Device, DeviceExportData> {
private final DeviceService deviceService;
private final DeviceCredentialsService credentialsService;
@Override
protected void setOwner(TenantId tenantId, Device device, IdProvider idProvider) {
device.setTenantId(tenantId);
device.setCustomerId(idProvider.getInternalId(device.getCustomerId()));
}
@Override
protected Device prepare(EntitiesImportCtx ctx, Device device, Device old, DeviceExportData exportData, IdProvider idProvider) {
device.setDeviceProfileId(idProvider.getInternalId(device.getDeviceProfileId()));
device.setFirmwareId(getOldEntityField(old, Device::getFirmwareId));
device.setSoftwareId(getOldEntityField(old, Device::getSoftwareId));
return device;
}
@Override
protected Device deepCopy(Device d) {
return new Device(d);
}
@Override
protected void cleanupForComparison(Device e) {
super.cleanupForComparison(e);
if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) {
e.setCustomerId(null);
}
}
@Override
protected Device saveOrUpdate(EntitiesImportCtx ctx, Device device, DeviceExportData exportData, IdProvider idProvider) {
if (exportData.getCredentials() != null && ctx.isSaveCredentials()) {
exportData.getCredentials().setId(null);
exportData.getCredentials().setDeviceId(null);
return deviceService.saveDeviceWithCredentials(device, exportData.getCredentials());
} else {
return deviceService.saveDevice(device);
}
}
@Override
protected boolean updateRelatedEntitiesIfUnmodified(EntitiesImportCtx ctx, Device prepared, DeviceExportData exportData, IdProvider idProvider) {
boolean updated = super.updateRelatedEntitiesIfUnmodified(ctx, prepared, exportData, idProvider);
var credentials = exportData.getCredentials();
if (credentials != null && ctx.isSaveCredentials()) {
var existing = credentialsService.findDeviceCredentialsByDeviceId(ctx.getTenantId(), prepared.getId());
credentials.setId(existing.getId());
credentials.setDeviceId(prepared.getId());
if (!existing.equals(credentials)) {
credentialsService.updateDeviceCredentials(ctx.getTenantId(), credentials);
updated = true;
}
}
return updated;
}
@Override
protected void onEntitySaved(SecurityUser user, Device savedDevice, Device oldDevice) throws ThingsboardException {
super.onEntitySaved(user, savedDevice, oldDevice);
clusterService.onDeviceUpdated(savedDevice, oldDevice);
}
@Override
public EntityType getEntityType() {
return EntityType.DEVICE;
}
}

93
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java

@ -0,0 +1,93 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import java.util.Objects;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class DeviceProfileImportService extends BaseEntityImportService<DeviceProfileId, DeviceProfile, EntityExportData<DeviceProfile>> {
private final DeviceProfileService deviceProfileService;
private final OtaPackageStateService otaPackageStateService;
@Override
protected void setOwner(TenantId tenantId, DeviceProfile deviceProfile, IdProvider idProvider) {
deviceProfile.setTenantId(tenantId);
}
@Override
protected DeviceProfile prepare(EntitiesImportCtx ctx, DeviceProfile deviceProfile, DeviceProfile old, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
deviceProfile.setDefaultRuleChainId(idProvider.getInternalId(deviceProfile.getDefaultRuleChainId()));
deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId()));
deviceProfile.setFirmwareId(getOldEntityField(old, DeviceProfile::getFirmwareId));
deviceProfile.setSoftwareId(getOldEntityField(old, DeviceProfile::getSoftwareId));
return deviceProfile;
}
@Override
protected DeviceProfile saveOrUpdate(EntitiesImportCtx ctx, DeviceProfile deviceProfile, EntityExportData<DeviceProfile> exportData, IdProvider idProvider) {
return deviceProfileService.saveDeviceProfile(deviceProfile);
}
@Override
protected void onEntitySaved(SecurityUser user, DeviceProfile savedDeviceProfile, DeviceProfile oldDeviceProfile) throws ThingsboardException {
super.onEntitySaved(user, savedDeviceProfile, oldDeviceProfile);
clusterService.onDeviceProfileChange(savedDeviceProfile, null);
clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(),
oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedDeviceProfile.getId(),
oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED);
otaPackageStateService.update(savedDeviceProfile,
oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()),
oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId()));
}
@Override
protected DeviceProfile deepCopy(DeviceProfile deviceProfile) {
return new DeviceProfile(deviceProfile);
}
@Override
protected void cleanupForComparison(DeviceProfile deviceProfile) {
super.cleanupForComparison(deviceProfile);
deviceProfile.setFirmwareId(null);
deviceProfile.setSoftwareId(null);
}
@Override
public EntityType getEntityType() {
return EntityType.DEVICE_PROFILE;
}
}

91
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java

@ -0,0 +1,91 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class EntityViewImportService extends BaseEntityImportService<EntityViewId, EntityView, EntityExportData<EntityView>> {
private final EntityViewService entityViewService;
@Lazy
@Autowired
private TbEntityViewService tbEntityViewService;
@Override
protected void setOwner(TenantId tenantId, EntityView entityView, IdProvider idProvider) {
entityView.setTenantId(tenantId);
entityView.setCustomerId(idProvider.getInternalId(entityView.getCustomerId()));
}
@Override
protected EntityView prepare(EntitiesImportCtx ctx, EntityView entityView, EntityView old, EntityExportData<EntityView> exportData, IdProvider idProvider) {
entityView.setEntityId(idProvider.getInternalId(entityView.getEntityId()));
return entityView;
}
@Override
protected EntityView saveOrUpdate(EntitiesImportCtx ctx, EntityView entityView, EntityExportData<EntityView> exportData, IdProvider idProvider) {
return entityViewService.saveEntityView(entityView);
}
@Override
protected void onEntitySaved(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException {
tbEntityViewService.updateEntityViewAttributes(user, savedEntityView, oldEntityView);
super.onEntitySaved(user, savedEntityView, oldEntityView);
if (oldEntityView != null) {
entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedEntityView.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
protected EntityView deepCopy(EntityView entityView) {
return new EntityView(entityView);
}
@Override
protected void cleanupForComparison(EntityView e) {
super.cleanupForComparison(e);
if (e.getCustomerId() != null && e.getCustomerId().isNullUid()) {
e.setCustomerId(null);
}
}
@Override
public EntityType getEntityType() {
return EntityType.ENTITY_VIEW;
}
}

20
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ImportServiceException.java

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
public class ImportServiceException extends RuntimeException{
private static final long serialVersionUID = -4932715239522125041L;
}

30
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/MissingEntityException.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.Getter;
import org.thingsboard.server.common.data.id.EntityId;
public class MissingEntityException extends ImportServiceException {
private static final long serialVersionUID = 3669135386955906022L;
@Getter
private final EntityId entityId;
public MissingEntityException(EntityId entityId) {
this.entityId = entityId;
}
}

152
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@ -0,0 +1,152 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.common.data.rule.RuleChainMetaData;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.rule.RuleNodeDao;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import org.thingsboard.common.util.RegexUtils;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@Slf4j
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class RuleChainImportService extends BaseEntityImportService<RuleChainId, RuleChain, RuleChainExportData> {
private static final LinkedHashSet<EntityType> HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET));
private final RuleChainService ruleChainService;
private final RuleNodeDao ruleNodeDao;
@Override
protected void setOwner(TenantId tenantId, RuleChain ruleChain, IdProvider idProvider) {
ruleChain.setTenantId(tenantId);
}
@Override
protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain, IdProvider idProvider) {
RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain, idProvider);
if (existingRuleChain == null && ctx.isFindExistingByName()) {
existingRuleChain = ruleChainService.findTenantRuleChainsByTypeAndName(ctx.getTenantId(), ruleChain.getType(), ruleChain.getName()).stream().findFirst().orElse(null);
}
return existingRuleChain;
}
@Override
protected RuleChain prepare(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChain old, RuleChainExportData exportData, IdProvider idProvider) {
RuleChainMetaData metaData = exportData.getMetaData();
List<RuleNode> ruleNodes = Optional.ofNullable(metaData.getNodes()).orElse(Collections.emptyList());
if (old != null) {
List<RuleNodeId> nodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList());
List<RuleNode> existing = ruleNodeDao.findByExternalIds(old.getId(), nodeIds);
existing.forEach(node -> ctx.putInternalId(node.getExternalId(), node.getId()));
ruleNodes.forEach(node -> {
node.setRuleChainId(old.getId());
node.setExternalId(node.getId());
node.setId((RuleNodeId) ctx.getInternalId(node.getId()));
});
} else {
ruleNodes.forEach(node -> {
node.setRuleChainId(null);
node.setExternalId(node.getId());
node.setId(null);
});
}
ruleNodes.forEach(ruleNode -> replaceIdsRecursively(ctx, idProvider, ruleNode.getConfiguration(), Collections.emptySet(), HINTS));
Optional.ofNullable(metaData.getRuleChainConnections()).orElse(Collections.emptyList())
.forEach(ruleChainConnectionInfo -> {
ruleChainConnectionInfo.setTargetRuleChainId(idProvider.getInternalId(ruleChainConnectionInfo.getTargetRuleChainId(), false));
});
if (ruleChain.getFirstRuleNodeId() != null) {
ruleChain.setFirstRuleNodeId((RuleNodeId) ctx.getInternalId(ruleChain.getFirstRuleNodeId()));
}
return ruleChain;
}
@Override
protected RuleChain saveOrUpdate(EntitiesImportCtx ctx, RuleChain ruleChain, RuleChainExportData exportData, IdProvider idProvider) {
ruleChain = ruleChainService.saveRuleChain(ruleChain);
if (ctx.isFinalImportAttempt() || ctx.getCurrentImportResult().isUpdatedAllExternalIds()) {
exportData.getMetaData().setRuleChainId(ruleChain.getId());
ruleChainService.saveRuleChainMetaData(ctx.getTenantId(), exportData.getMetaData());
return ruleChainService.findRuleChainById(ctx.getTenantId(), ruleChain.getId());
} else {
return ruleChain;
}
}
@Override
protected boolean compare(EntitiesImportCtx ctx, RuleChainExportData exportData, RuleChain prepared, RuleChain existing) {
boolean different = super.compare(ctx, exportData, prepared, existing);
if (!different) {
RuleChainMetaData newMD = exportData.getMetaData();
RuleChainMetaData existingMD = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), prepared.getId());
existingMD.setRuleChainId(null);
different = newMD.equals(existingMD);
}
return different;
}
@Override
protected void onEntitySaved(SecurityUser user, RuleChain savedRuleChain, RuleChain oldRuleChain) throws ThingsboardException {
super.onEntitySaved(user, savedRuleChain, oldRuleChain);
if (savedRuleChain.getType() == RuleChainType.CORE) {
clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(),
oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
} else if (savedRuleChain.getType() == RuleChainType.EDGE && oldRuleChain != null) {
entityActionService.sendEntityNotificationMsgToEdgeService(user.getTenantId(), savedRuleChain.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
protected RuleChain deepCopy(RuleChain ruleChain) {
return new RuleChain(ruleChain);
}
@Override
public EntityType getEntityType() {
return EntityType.RULE_CHAIN;
}
}

106
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java

@ -0,0 +1,106 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetsBundleId;
import org.thingsboard.server.common.data.sync.ie.WidgetsBundleExportData;
import org.thingsboard.server.common.data.widget.BaseWidgetType;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetTypeInfo;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import java.util.Map;
import java.util.stream.Collectors;
@Service
@TbCoreComponent
@RequiredArgsConstructor
public class WidgetsBundleImportService extends BaseEntityImportService<WidgetsBundleId, WidgetsBundle, WidgetsBundleExportData> {
private final WidgetsBundleService widgetsBundleService;
private final WidgetTypeService widgetTypeService;
@Override
protected void setOwner(TenantId tenantId, WidgetsBundle widgetsBundle, IdProvider idProvider) {
widgetsBundle.setTenantId(tenantId);
}
@Override
protected WidgetsBundle prepare(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundle old, WidgetsBundleExportData exportData, IdProvider idProvider) {
return widgetsBundle;
}
@Override
protected WidgetsBundle saveOrUpdate(EntitiesImportCtx ctx, WidgetsBundle widgetsBundle, WidgetsBundleExportData exportData, IdProvider idProvider) {
WidgetsBundle savedWidgetsBundle = widgetsBundleService.saveWidgetsBundle(widgetsBundle);
if (widgetsBundle.getId() == null) {
for (WidgetTypeDetails widget : exportData.getWidgets()) {
widget.setId(null);
widget.setTenantId(ctx.getTenantId());
widget.setBundleAlias(savedWidgetsBundle.getAlias());
widgetTypeService.saveWidgetType(widget);
}
} else {
Map<String, WidgetTypeInfo> existingWidgets = widgetTypeService.findWidgetTypesInfosByTenantIdAndBundleAlias(ctx.getTenantId(), savedWidgetsBundle.getAlias()).stream()
.collect(Collectors.toMap(BaseWidgetType::getAlias, w -> w));
for (WidgetTypeDetails widget : exportData.getWidgets()) {
WidgetTypeInfo existingWidget;
if ((existingWidget = existingWidgets.remove(widget.getAlias())) != null) {
widget.setId(existingWidget.getId());
widget.setCreatedTime(existingWidget.getCreatedTime());
} else {
widget.setId(null);
}
widget.setTenantId(ctx.getTenantId());
widget.setBundleAlias(savedWidgetsBundle.getAlias());
widgetTypeService.saveWidgetType(widget);
}
existingWidgets.values().stream()
.map(BaseWidgetType::getId)
.forEach(widgetTypeId -> widgetTypeService.deleteWidgetType(ctx.getTenantId(), widgetTypeId));
}
return savedWidgetsBundle;
}
@Override
protected void onEntitySaved(SecurityUser user, WidgetsBundle savedWidgetsBundle, WidgetsBundle oldWidgetsBundle) throws ThingsboardException {
super.onEntitySaved(user, savedWidgetsBundle, oldWidgetsBundle);
entityNotificationService.notifySendMsgToEdgeService(user.getTenantId(), savedWidgetsBundle.getId(),
oldWidgetsBundle == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED);
}
@Override
protected WidgetsBundle deepCopy(WidgetsBundle widgetsBundle) {
return new WidgetsBundle(widgetsBundle);
}
@Override
public EntityType getEntityType() {
return EntityType.WIDGETS_BUNDLE;
}
}

554
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -0,0 +1,554 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.TbStopWatch;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityLoadError;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.AutoVersionCreateConfig;
import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.EntityTypeVersionCreateConfig;
import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException;
import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService;
import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx;
import org.thingsboard.server.service.sync.vc.data.ReimportTask;
import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.function.Function;
import java.util.stream.Collectors;
import static com.google.common.util.concurrent.Futures.transform;
import static com.google.common.util.concurrent.Futures.transformAsync;
@Service
@TbCoreComponent
@RequiredArgsConstructor
@Slf4j
public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService {
private final TbRepositorySettingsService repositorySettingsService;
private final TbAutoCommitSettingsService autoCommitSettingsService;
private final GitVersionControlQueueService gitServiceQueue;
private final EntitiesExportImportService exportImportService;
private final ExportableEntitiesService exportableEntitiesService;
private final TbNotificationEntityService entityNotificationService;
private final TransactionTemplate transactionTemplate;
private ListeningExecutorService executor;
@Value("${vc.thread_pool_size:4}")
private int threadPoolSize;
@PostConstruct
public void init() {
executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class));
}
@PreDestroy
public void shutdown() {
if (executor != null) {
executor.shutdownNow();
}
}
@SuppressWarnings("UnstableApiUsage")
@Override
public ListenableFuture<VersionCreationResult> saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception {
var pendingCommit = gitServiceQueue.prepareCommit(user, request);
return transformAsync(pendingCommit, commit -> {
List<ListenableFuture<Void>> gitFutures = new ArrayList<>();
switch (request.getType()) {
case SINGLE_ENTITY: {
handleSingleEntityRequest(new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request));
break;
}
case COMPLEX: {
handleComplexRequest(new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request));
break;
}
}
return transformAsync(Futures.allAsList(gitFutures), success -> gitServiceQueue.push(commit), executor);
}, executor);
}
private void handleSingleEntityRequest(SimpleEntitiesExportCtx ctx) throws Exception {
ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId()));
}
private void handleComplexRequest(ComplexEntitiesExportCtx parentCtx) {
ComplexVersionCreateRequest request = parentCtx.getRequest();
request.getEntityTypes().forEach((entityType, config) -> {
EntityTypeExportCtx ctx = new EntityTypeExportCtx(parentCtx, config, request.getSyncStrategy(), entityType);
if (ctx.isOverwrite()) {
ctx.add(gitServiceQueue.deleteAll(ctx.getCommit(), entityType));
}
if (config.isAllEntities()) {
DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink)
, 100, entity -> {
try {
ctx.add(saveEntityData(ctx, entity.getId()));
} catch (Exception e) {
throw new RuntimeException(e);
}
});
} else {
for (UUID entityId : config.getEntityIds()) {
try {
ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId)));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
});
}
private ListenableFuture<Void> saveEntityData(EntitiesExportCtx<?> ctx, EntityId entityId) throws Exception {
EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(ctx, entityId);
return gitServiceQueue.addToCommit(ctx.getCommit(), entityData);
}
@Override
public ListenableFuture<PageData<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, externalId, pageLink);
}
@Override
public ListenableFuture<PageData<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, entityType, pageLink);
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception {
return gitServiceQueue.listVersions(tenantId, branch, pageLink);
}
@Override
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception {
return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId, entityType);
}
@Override
public ListenableFuture<List<VersionedEntityInfo>> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
return gitServiceQueue.listEntitiesAtVersion(tenantId, branch, versionId);
}
@SuppressWarnings({"UnstableApiUsage", "rawtypes"})
@Override
public ListenableFuture<VersionLoadResult> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception {
switch (request.getType()) {
case SINGLE_ENTITY: {
SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request;
VersionLoadConfig config = versionLoadRequest.getConfig();
ListenableFuture<EntityExportData> future = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId());
return Futures.transform(future, entityData -> doInTemplate(user, request, ctx -> loadSingleEntity(ctx, config, entityData)), executor);
}
case ENTITY_TYPE: {
EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request;
return executor.submit(() -> doInTemplate(user, request, ctx -> loadMultipleEntities(ctx, versionLoadRequest)));
}
default:
throw new IllegalArgumentException("Unsupported version load request");
}
}
private <R> VersionLoadResult doInTemplate(SecurityUser user, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
try {
EntitiesImportCtx ctx = new EntitiesImportCtx(user, request.getVersionId());
VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx));
try {
for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
throwingRunnable.run();
}
} catch (ThingsboardException e) {
throw new RuntimeException(e);
}
return result;
} catch (LoadEntityException e) {
return onError(e.getData(), e.getCause());
} catch (Exception e) {
log.info("[{}] Failed to process request [{}] due to: ", user.getTenantId(), request, e);
throw e;
}
}
private VersionLoadResult loadSingleEntity(EntitiesImportCtx ctx, VersionLoadConfig config, EntityExportData entityData) {
try {
ctx.setSettings(EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.findExistingByName(false)
.build());
ctx.setFinalImportAttempt(true);
EntityImportResult<?> importResult = exportImportService.importEntity(ctx, entityData);
exportImportService.saveReferencesAndRelations(ctx);
return VersionLoadResult.success(EntityTypeLoadResult.builder()
.entityType(importResult.getEntityType())
.created(importResult.getOldEntity() == null ? 1 : 0)
.updated(importResult.getOldEntity() != null ? 1 : 0)
.deleted(0)
.build());
} catch (Exception e) {
throw new LoadEntityException(entityData, e);
}
}
@SneakyThrows
private VersionLoadResult loadMultipleEntities(EntitiesImportCtx ctx, EntityTypeVersionLoadRequest request) {
var sw = TbStopWatch.create("before");
List<EntityType> entityTypes = request.getEntityTypes().keySet().stream()
.sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList());
for (EntityType entityType : entityTypes) {
log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
sw.startNew("Entities " + entityType.name());
ctx.setSettings(getEntityImportSettings(request, entityType));
importEntities(ctx, entityType);
}
sw.startNew("Reimport");
reimport(ctx);
sw.startNew("Remove Others");
request.getEntityTypes().keySet().stream()
.filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
.sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
.forEach(entityType -> removeOtherEntities(ctx, entityType));
sw.startNew("References and Relations");
exportImportService.saveReferencesAndRelations(ctx);
sw.stop();
for (var task : sw.getTaskInfo()) {
log.info("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
}
log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values()));
}
private EntityImportSettings getEntityImportSettings(EntityTypeVersionLoadRequest request, EntityType entityType) {
var config = request.getEntityTypes().get(entityType);
return EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.findExistingByName(config.isFindExistingEntityByName())
.build();
}
@SuppressWarnings({"rawtypes", "unchecked"})
private void importEntities(EntitiesImportCtx ctx, EntityType entityType) {
int limit = 100;
int offset = 0;
List<EntityExportData> entityDataList;
do {
try {
entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
for (EntityExportData entityData : entityDataList) {
EntityExportData reimportBackup = JacksonUtil.clone(entityData);
log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
EntityImportResult<?> importResult;
try {
importResult = exportImportService.importEntity(ctx, entityData);
} catch (Exception e) {
throw new LoadEntityException(entityData, e);
}
if (!importResult.isUpdatedAllExternalIds()) {
ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(reimportBackup, ctx.getSettings()));
continue;
}
registerResult(ctx, entityType, importResult);
ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>())
.add(importResult.getSavedEntity().getId());
}
offset += limit;
} while (entityDataList.size() == limit);
}
@SuppressWarnings({"rawtypes", "unchecked"})
private void reimport(EntitiesImportCtx ctx) {
ctx.setFinalImportAttempt(true);
ctx.getToReimport().forEach((externalId, task) -> {
try {
EntityExportData entityData = task.getData();
var settings = task.getSettings();
ctx.setSettings(settings);
EntityImportResult<?> importResult = exportImportService.importEntity(ctx, entityData);
registerResult(ctx, externalId.getEntityType(), importResult);
ctx.getImportedEntities().computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>())
.add(importResult.getSavedEntity().getId());
} catch (Exception e) {
throw new RuntimeException(e);
}
});
}
private void removeOtherEntities(EntitiesImportCtx ctx, EntityType entityType) {
DaoUtil.processInBatches(pageLink -> {
return exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink);
}, 100, entity -> {
if (ctx.getImportedEntities().get(entityType) == null || !ctx.getImportedEntities().get(entityType).contains(entity.getId())) {
exportableEntitiesService.removeById(ctx.getTenantId(), entity.getId());
ctx.addEventCallback(() -> {
entityNotificationService.notifyDeleteEntity(ctx.getTenantId(), entity.getId(),
entity, null, ActionType.DELETED, null, ctx.getUser());
});
ctx.registerDeleted(entityType);
}
});
}
private VersionLoadResult onError(EntityExportData<?> entityData, Throwable e) {
return analyze(e, entityData).orElseThrow(() -> new RuntimeException(e));
}
private Optional<VersionLoadResult> analyze(Throwable e, EntityExportData<?> entityData) {
if (e == null) {
return Optional.empty();
} else {
if (e instanceof DeviceCredentialsValidationException) {
return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId())));
} else if (e instanceof MissingEntityException) {
return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId())));
} else {
return analyze(e.getCause(), entityData);
}
}
}
@Override
public ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception {
HasId<EntityId> entity = exportableEntitiesService.findEntityByTenantIdAndId(user.getTenantId(), entityId);
if (!(entity instanceof ExportableEntity)) throw new IllegalArgumentException("Unsupported entity type");
EntityId externalId = ((ExportableEntity<EntityId>) entity).getExternalId();
if (externalId == null) externalId = entityId;
return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId),
otherVersion -> {
SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder()
.exportRelations(otherVersion.hasRelations())
.exportAttributes(otherVersion.hasAttributes())
.exportCredentials(otherVersion.hasCredentials())
.build());
EntityExportData<?> currentVersion = exportImportService.exportEntity(ctx, entityId);
return transform(gitServiceQueue.getContentsDiff(user.getTenantId(),
JacksonUtil.toPrettyString(currentVersion.sort()),
JacksonUtil.toPrettyString(otherVersion.sort())),
rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor());
}, MoreExecutors.directExecutor());
}
@Override
public ListenableFuture<EntityDataInfo> getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId) {
return Futures.transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, entityId),
entity -> new EntityDataInfo(entity.hasRelations(), entity.hasAttributes(), entity.hasCredentials()), MoreExecutors.directExecutor());
}
@Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception {
return gitServiceQueue.listBranches(tenantId);
}
@Override
public RepositorySettings getVersionControlSettings(TenantId tenantId) {
return repositorySettingsService.get(tenantId);
}
@Override
public ListenableFuture<RepositorySettings> saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings) {
var restoredSettings = this.repositorySettingsService.restore(tenantId, versionControlSettings);
try {
var future = gitServiceQueue.initRepository(tenantId, restoredSettings);
return Futures.transform(future, f -> repositorySettingsService.save(tenantId, restoredSettings), MoreExecutors.directExecutor());
} catch (Exception e) {
log.debug("{} Failed to init repository: {}", tenantId, versionControlSettings, e);
throw new RuntimeException("Failed to init repository!", e);
}
}
@Override
public ListenableFuture<Void> deleteVersionControlSettings(TenantId tenantId) throws Exception {
if (repositorySettingsService.delete(tenantId)) {
return gitServiceQueue.clearRepository(tenantId);
} else {
return Futures.immediateFuture(null);
}
}
@Override
public ListenableFuture<Void> checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws ThingsboardException {
settings = this.repositorySettingsService.restore(tenantId, settings);
try {
return gitServiceQueue.testRepository(tenantId, settings);
} catch (Exception e) {
throw new ThingsboardException(String.format("Unable to access repository: %s", getCauseMessage(e)),
ThingsboardErrorCode.GENERAL);
}
}
@Override
public ListenableFuture<VersionCreationResult> autoCommit(SecurityUser user, EntityId entityId) throws Exception {
var repositorySettings = repositorySettingsService.get(user.getTenantId());
if (repositorySettings == null) {
return Futures.immediateFuture(null);
}
var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId());
if (autoCommitSettings == null) {
return Futures.immediateFuture(null);
}
var entityType = entityId.getEntityType();
AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType);
if (autoCommitConfig == null) {
return Futures.immediateFuture(null);
}
SingleEntityVersionCreateRequest vcr = new SingleEntityVersionCreateRequest();
var autoCommitBranchName = autoCommitConfig.getBranch();
if (StringUtils.isEmpty(autoCommitBranchName)) {
autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits";
}
vcr.setBranch(autoCommitBranchName);
vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000));
vcr.setEntityId(entityId);
vcr.setConfig(autoCommitConfig);
return saveEntitiesVersion(user, vcr);
}
@Override
public ListenableFuture<VersionCreationResult> autoCommit(SecurityUser user, EntityType entityType, List<UUID> entityIds) throws Exception {
var repositorySettings = repositorySettingsService.get(user.getTenantId());
if (repositorySettings == null) {
return Futures.immediateFuture(null);
}
var autoCommitSettings = autoCommitSettingsService.get(user.getTenantId());
if (autoCommitSettings == null) {
return Futures.immediateFuture(null);
}
AutoVersionCreateConfig autoCommitConfig = autoCommitSettings.get(entityType);
if (autoCommitConfig == null) {
return Futures.immediateFuture(null);
}
var autoCommitBranchName = autoCommitConfig.getBranch();
if (StringUtils.isEmpty(autoCommitBranchName)) {
autoCommitBranchName = StringUtils.isNotEmpty(repositorySettings.getDefaultBranch()) ? repositorySettings.getDefaultBranch() : "auto-commits";
}
ComplexVersionCreateRequest vcr = new ComplexVersionCreateRequest();
vcr.setBranch(autoCommitBranchName);
vcr.setVersionName("auto-commit at " + Instant.ofEpochSecond(System.currentTimeMillis() / 1000));
vcr.setSyncStrategy(SyncStrategy.MERGE);
EntityTypeVersionCreateConfig vcrConfig = new EntityTypeVersionCreateConfig();
vcrConfig.setEntityIds(entityIds);
vcr.setEntityTypes(Collections.singletonMap(entityType, vcrConfig));
return saveEntitiesVersion(user, vcr);
}
private String getCauseMessage(Exception e) {
String message;
if (e.getCause() != null && StringUtils.isNotEmpty(e.getCause().getMessage())) {
message = e.getCause().getMessage();
} else {
message = e.getMessage();
}
return message;
}
private void registerResult(EntitiesImportCtx ctx, EntityType entityType, EntityImportResult<?> importResult) {
if (importResult.isCreated()) {
ctx.registerResult(entityType, true);
} else if (importResult.isUpdated() || importResult.isUpdatedRelatedEntities()) {
ctx.registerResult(entityType, false);
}
}
}

526
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java

@ -0,0 +1,526 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.protobuf.ByteString;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntitiesContentRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.EntityContentRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GenericRepositoryRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListEntitiesRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ListVersionsRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.PrepareMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.service.sync.vc.data.ContentsDiffGitRequest;
import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest;
import org.thingsboard.server.service.sync.vc.data.EntityContentGitRequest;
import org.thingsboard.server.service.sync.vc.data.ListBranchesGitRequest;
import org.thingsboard.server.service.sync.vc.data.ListEntitiesGitRequest;
import org.thingsboard.server.service.sync.vc.data.ListVersionsGitRequest;
import org.thingsboard.server.service.sync.vc.data.PendingGitRequest;
import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest;
import org.thingsboard.server.service.sync.vc.data.VoidGitRequest;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@TbCoreComponent
@Service
@Slf4j
public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService {
private final TbServiceInfoProvider serviceInfoProvider;
private final TbClusterService clusterService;
private final DataDecodingEncodingService encodingService;
private final DefaultEntitiesVersionControlService entitiesVersionControlService;
private final SchedulerComponent scheduler;
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>();
@Value("${queue.vc.request-timeout:60000}")
private int requestTimeout;
public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService,
DataDecodingEncodingService encodingService,
@Lazy DefaultEntitiesVersionControlService entitiesVersionControlService,
SchedulerComponent scheduler) {
this.serviceInfoProvider = serviceInfoProvider;
this.clusterService = clusterService;
this.encodingService = encodingService;
this.entitiesVersionControlService = entitiesVersionControlService;
this.scheduler = scheduler;
}
@Override
public ListenableFuture<CommitGitRequest> prepareCommit(User user, VersionCreateRequest request) {
SettableFuture<CommitGitRequest> future = SettableFuture.create();
CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request);
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build()
).build(), wrap(future, commit));
return future;
}
@Override
public ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData) {
SettableFuture<Void> future = SettableFuture.create();
String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId());
String entityDataJson = JacksonUtil.toPrettyString(entityData.sort());
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setAddMsg(
TransportProtos.AddMsg.newBuilder()
.setRelativePath(path).setEntityDataJson(entityDataJson).build()
).build()
).build(), wrap(future, null));
return future;
}
@Override
public ListenableFuture<Void> deleteAll(CommitGitRequest commit, EntityType entityType) {
SettableFuture<Void> future = SettableFuture.create();
String path = getRelativePath(entityType, null);
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setDeleteMsg(
TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build()
).build()
).build(), wrap(future, null));
return future;
}
@Override
public ListenableFuture<VersionCreationResult> push(CommitGitRequest commit) {
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setPushMsg(
TransportProtos.PushMsg.newBuilder().build()
).build()
).build(), wrap(commit.getFuture()));
return commit.getFuture();
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) {
return listVersions(tenantId,
applyPageLinkParameters(
ListVersionsRequestMsg.newBuilder()
.setBranchName(branch),
pageLink
).build());
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) {
return listVersions(tenantId,
applyPageLinkParameters(
ListVersionsRequestMsg.newBuilder()
.setBranchName(branch)
.setEntityType(entityType.name()),
pageLink
).build());
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink) {
return listVersions(tenantId,
applyPageLinkParameters(
ListVersionsRequestMsg.newBuilder()
.setBranchName(branch)
.setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits()),
pageLink
).build());
}
private ListVersionsRequestMsg.Builder applyPageLinkParameters(ListVersionsRequestMsg.Builder builder, PageLink pageLink) {
builder.setPageSize(pageLink.getPageSize())
.setPage(pageLink.getPage());
if (pageLink.getTextSearch() != null) {
builder.setTextSearch(pageLink.getTextSearch());
}
if (pageLink.getSortOrder() != null) {
if (pageLink.getSortOrder().getProperty() != null) {
builder.setSortProperty(pageLink.getSortOrder().getProperty());
}
if (pageLink.getSortOrder().getDirection() != null) {
builder.setSortDirection(pageLink.getSortOrder().getDirection().name());
}
}
return builder;
}
private ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, ListVersionsRequestMsg requestMsg) {
ListVersionsGitRequest request = new ListVersionsGitRequest(tenantId);
return sendRequest(request, builder -> builder.setListVersionRequest(requestMsg));
}
@Override
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) {
return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder()
.setBranchName(branch)
.setVersionId(versionId)
.setEntityType(entityType.name())
.build());
}
@Override
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) {
return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder()
.setBranchName(branch)
.setVersionId(versionId)
.build());
}
private ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, TransportProtos.ListEntitiesRequestMsg requestMsg) {
ListEntitiesGitRequest request = new ListEntitiesGitRequest(tenantId);
return sendRequest(request, builder -> builder.setListEntitiesRequest(requestMsg));
}
@Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) {
ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId);
return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()));
}
@Override
public ListenableFuture<List<EntityVersionsDiff>> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2) {
String path = entityType != null ? getRelativePath(entityType, externalId) : "";
VersionsDiffGitRequest request = new VersionsDiffGitRequest(tenantId, path, versionId1, versionId2);
return sendRequest(request, builder -> builder.setVersionsDiffRequest(TransportProtos.VersionsDiffRequestMsg.newBuilder()
.setPath(request.getPath())
.setVersionId1(request.getVersionId1())
.setVersionId2(request.getVersionId2())
.build()));
}
@Override
public ListenableFuture<String> getContentsDiff(TenantId tenantId, String content1, String content2) {
ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2);
return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder()
.setContent1(content1)
.setContent2(content2)));
}
@Override
@SuppressWarnings("rawtypes")
public ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId) {
EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId);
registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build()
, wrap(request.getFuture()));
return request.getFuture();
}
private <T> void registerAndSend(PendingGitRequest<T> request,
Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction, TbQueueCallback callback) {
registerAndSend(request, enrichFunction, null, callback);
}
private <T> void registerAndSend(PendingGitRequest<T> request,
Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction, RepositorySettings settings, TbQueueCallback callback) {
if (!request.getFuture().isDone()) {
pendingRequestMap.putIfAbsent(request.getRequestId(), request);
var requestBody = enrichFunction.apply(newRequestProto(request, settings));
log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody);
clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback);
request.setTimeoutTask(scheduler.schedule(() -> {
processTimeout(request.getRequestId());
}, requestTimeout, TimeUnit.MILLISECONDS));
} else {
throw new RuntimeException("Future is already done!");
}
}
private <T> ListenableFuture<T> sendRequest(PendingGitRequest<T> request, Consumer<ToVersionControlServiceMsg.Builder> enrichFunction) {
registerAndSend(request, builder -> {
enrichFunction.accept(builder);
return builder.build();
}, wrap(request.getFuture()));
return request.getFuture();
}
@Override
@SuppressWarnings("rawtypes")
public ListenableFuture<List<EntityExportData>> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) {
EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType);
registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityType.name())
.setOffset(offset)
.setLimit(limit)
).build()
, wrap(request.getFuture()));
return request.getFuture();
}
@Override
public ListenableFuture<Void> initRepository(TenantId tenantId, RepositorySettings settings) {
VoidGitRequest request = new VoidGitRequest(tenantId);
registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build()
, settings, wrap(request.getFuture()));
return request.getFuture();
}
@Override
public ListenableFuture<Void> testRepository(TenantId tenantId, RepositorySettings settings) {
VoidGitRequest request = new VoidGitRequest(tenantId);
registerAndSend(request, builder -> builder
.setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build()
, settings, wrap(request.getFuture()));
return request.getFuture();
}
@Override
public ListenableFuture<Void> clearRepository(TenantId tenantId) {
ClearRepositoryGitRequest request = new ClearRepositoryGitRequest(tenantId);
registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build()
, wrap(request.getFuture()));
return request.getFuture();
}
@Override
public void processResponse(VersionControlResponseMsg vcResponseMsg) {
UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB());
PendingGitRequest<?> request = pendingRequestMap.remove(requestId);
if (request == null) {
log.debug("[{}] received stale response: {}", requestId, vcResponseMsg);
return;
} else {
log.debug("[{}] processing response: {}", requestId, vcResponseMsg);
request.getTimeoutTask().cancel(true);
}
var future = request.getFuture();
if (!StringUtils.isEmpty(vcResponseMsg.getError())) {
future.setException(new RuntimeException(vcResponseMsg.getError()));
} else {
if (vcResponseMsg.hasGenericResponse()) {
future.set(null);
} else if (vcResponseMsg.hasCommitResponse()) {
var commitResponse = vcResponseMsg.getCommitResponse();
var commitResult = new VersionCreationResult();
if (commitResponse.getTs() > 0) {
commitResult.setVersion(new EntityVersion(commitResponse.getTs(), commitResponse.getCommitId(), commitResponse.getName(), commitResponse.getAuthor()));
}
commitResult.setAdded(commitResponse.getAdded());
commitResult.setRemoved(commitResponse.getRemoved());
commitResult.setModified(commitResponse.getModified());
((CommitGitRequest) request).getFuture().set(commitResult);
} else if (vcResponseMsg.hasListBranchesResponse()) {
var listBranchesResponse = vcResponseMsg.getListBranchesResponse();
((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList());
} else if (vcResponseMsg.hasListEntitiesResponse()) {
var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse();
((ListEntitiesGitRequest) request).getFuture().set(
listEntitiesResponse.getEntitiesList().stream().map(this::getVersionedEntityInfo).collect(Collectors.toList()));
} else if (vcResponseMsg.hasListVersionsResponse()) {
var listVersionsResponse = vcResponseMsg.getListVersionsResponse();
((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse));
} else if (vcResponseMsg.hasEntityContentResponse()) {
var data = vcResponseMsg.getEntityContentResponse().getData();
((EntityContentGitRequest) request).getFuture().set(toData(data));
} else if (vcResponseMsg.hasEntitiesContentResponse()) {
var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList();
((EntitiesContentGitRequest) request).getFuture()
.set(dataList.stream().map(this::toData).collect(Collectors.toList()));
} else if (vcResponseMsg.hasVersionsDiffResponse()) {
TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse();
List<EntityVersionsDiff> entityVersionsDiffList = diffResponse.getDiffList().stream()
.map(diff -> EntityVersionsDiff.builder()
.externalId(EntityIdFactory.getByTypeAndUuid(EntityType.valueOf(diff.getEntityType()),
new UUID(diff.getEntityIdMSB(), diff.getEntityIdLSB())))
.entityDataAtVersion1(StringUtils.isNotEmpty(diff.getEntityDataAtVersion1()) ?
toData(diff.getEntityDataAtVersion1()) : null)
.entityDataAtVersion2(StringUtils.isNotEmpty(diff.getEntityDataAtVersion2()) ?
toData(diff.getEntityDataAtVersion2()) : null)
.rawDiff(diff.getRawDiff())
.build())
.collect(Collectors.toList());
((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList);
} else if (vcResponseMsg.hasContentsDiffResponse()) {
String diff = vcResponseMsg.getContentsDiffResponse().getDiff();
((ContentsDiffGitRequest) request).getFuture().set(diff);
}
}
}
private void processTimeout(UUID requestId) {
PendingGitRequest<?> pendingRequest = pendingRequestMap.remove(requestId);
if (pendingRequest != null) {
log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout);
pendingRequest.getFuture().setException(new TimeoutException("Request timed out"));
}
}
private PageData<EntityVersion> toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) {
var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList());
return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext());
}
private EntityVersion getEntityVersion(TransportProtos.EntityVersionProto proto) {
return new EntityVersion(proto.getTs(), proto.getId(), proto.getName(), proto.getAuthor());
}
private VersionedEntityInfo getVersionedEntityInfo(TransportProtos.VersionedEntityInfoProto proto) {
return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())));
}
@SuppressWarnings("rawtypes")
@SneakyThrows
private EntityExportData toData(String data) {
return JacksonUtil.fromString(data, EntityExportData.class);
}
private static <T> TbQueueCallback wrap(SettableFuture<T> future) {
return new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
}
@Override
public void onFailure(Throwable t) {
future.setException(t);
}
};
}
private static <T> TbQueueCallback wrap(SettableFuture<T> future, T value) {
return new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
future.set(value);
}
@Override
public void onFailure(Throwable t) {
future.setException(t);
}
};
}
private static String getRelativePath(EntityType entityType, EntityId entityId) {
String path = entityType.name().toLowerCase();
if (entityId != null) {
path += "/" + entityId + ".json";
}
return path;
}
private static PrepareMsg getCommitPrepareMsg(User user, VersionCreateRequest request) {
return PrepareMsg.newBuilder().setCommitMsg(request.getVersionName())
.setBranchName(request.getBranch()).setAuthorName(getAuthorName(user)).setAuthorEmail(user.getEmail()).build();
}
private static String getAuthorName(User user) {
List<String> parts = new ArrayList<>();
if (StringUtils.isNotBlank(user.getFirstName())) {
parts.add(user.getFirstName());
}
if (StringUtils.isNotBlank(user.getLastName())) {
parts.add(user.getLastName());
}
if (parts.isEmpty()) {
parts.add(user.getName());
}
return String.join(" ", parts);
}
private ToVersionControlServiceMsg.Builder newRequestProto(PendingGitRequest<?> request, RepositorySettings settings) {
var tenantId = request.getTenantId();
var requestId = request.getRequestId();
var builder = ToVersionControlServiceMsg.newBuilder()
.setNodeId(serviceInfoProvider.getServiceId())
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setRequestIdMSB(requestId.getMostSignificantBits())
.setRequestIdLSB(requestId.getLeastSignificantBits());
RepositorySettings vcSettings = settings;
if (vcSettings == null && request.requiresSettings()) {
vcSettings = entitiesVersionControlService.getVersionControlSettings(tenantId);
}
if (vcSettings != null) {
builder.setVcSettings(ByteString.copyFrom(encodingService.encode(vcSettings)));
} else if (request.requiresSettings()) {
throw new RuntimeException("No entity version control settings provisioned!");
}
return builder;
}
private CommitRequestMsg.Builder buildCommitRequest(CommitGitRequest commit) {
return CommitRequestMsg.newBuilder().setTxId(commit.getTxId().toString());
}
}

72
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java

@ -0,0 +1,72 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.List;
import java.util.UUID;
public interface EntitiesVersionControlService {
ListenableFuture<VersionCreationResult> saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception;
ListenableFuture<PageData<EntityVersion>> listEntityVersions(TenantId tenantId, String branch, EntityId externalId, PageLink pageLink) throws Exception;
ListenableFuture<PageData<EntityVersion>> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink) throws Exception;
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) throws Exception;
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception;
ListenableFuture<List<VersionedEntityInfo>> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
ListenableFuture<VersionLoadResult> loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception;
ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception;
ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception;
RepositorySettings getVersionControlSettings(TenantId tenantId);
ListenableFuture<RepositorySettings> saveVersionControlSettings(TenantId tenantId, RepositorySettings versionControlSettings);
ListenableFuture<Void> deleteVersionControlSettings(TenantId tenantId) throws Exception;
ListenableFuture<Void> checkVersionControlAccess(TenantId tenantId, RepositorySettings settings) throws Exception;
ListenableFuture<VersionCreationResult> autoCommit(SecurityUser user, EntityId entityId) throws Exception;
ListenableFuture<VersionCreationResult> autoCommit(SecurityUser user, EntityType entityType, List<UUID> entityIds) throws Exception;
ListenableFuture<EntityDataInfo> getEntityDataInfo(SecurityUser user, EntityId entityId, String versionId);
}

75
application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java

@ -0,0 +1,75 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import java.util.List;
public interface GitVersionControlQueueService {
ListenableFuture<CommitGitRequest> prepareCommit(User user, VersionCreateRequest request);
ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData);
ListenableFuture<Void> deleteAll(CommitGitRequest pendingCommit, EntityType entityType);
ListenableFuture<VersionCreationResult> push(CommitGitRequest commit);
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink);
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityType entityType, PageLink pageLink);
ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, EntityId entityId, PageLink pageLink);
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType);
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId);
ListenableFuture<List<String>> listBranches(TenantId tenantId);
ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId);
ListenableFuture<List<EntityExportData>> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit);
ListenableFuture<List<EntityVersionsDiff>> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2);
ListenableFuture<String> getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2);
ListenableFuture<Void> initRepository(TenantId tenantId, RepositorySettings settings);
ListenableFuture<Void> testRepository(TenantId tenantId, RepositorySettings settings);
ListenableFuture<Void> clearRepository(TenantId tenantId);
void processResponse(VersionControlResponseMsg vcResponseMsg);
}

32
application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import lombok.Getter;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
@SuppressWarnings("rawtypes")
public class LoadEntityException extends RuntimeException {
private static final long serialVersionUID = -1749719992370409504L;
@Getter
private final EntityExportData data;
public LoadEntityException(EntityExportData data, Throwable cause) {
super(cause);
this.data = data;
}
}

80
application/src/main/java/org/thingsboard/server/service/sync/vc/TbAbstractVersionControlSettingsService.java

@ -0,0 +1,80 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import java.io.Serializable;
public abstract class TbAbstractVersionControlSettingsService<T extends Serializable> {
private final String settingsKey;
private final AdminSettingsService adminSettingsService;
private final TbTransactionalCache<TenantId, T> cache;
private final Class<T> clazz;
public TbAbstractVersionControlSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache<TenantId, T> cache, Class<T> clazz, String settingsKey) {
this.adminSettingsService = adminSettingsService;
this.cache = cache;
this.clazz = clazz;
this.settingsKey = settingsKey;
}
public T get(TenantId tenantId) {
return cache.getAndPutInTransaction(tenantId, () -> {
AdminSettings adminSettings = adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, settingsKey);
if (adminSettings != null) {
try {
return JacksonUtil.convertValue(adminSettings.getJsonValue(), clazz);
} catch (Exception e) {
throw new RuntimeException("Failed to load " + settingsKey + " settings!", e);
}
}
return null;
}, true);
}
public T save(TenantId tenantId, T settings) {
AdminSettings adminSettings = adminSettingsService.findAdminSettingsByTenantIdAndKey(tenantId, settingsKey);
if (adminSettings == null) {
adminSettings = new AdminSettings();
adminSettings.setKey(settingsKey);
adminSettings.setTenantId(tenantId);
}
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings));
AdminSettings savedAdminSettings = adminSettingsService.saveAdminSettings(tenantId, adminSettings);
T savedSettings;
try {
savedSettings = JacksonUtil.convertValue(savedAdminSettings.getJsonValue(), clazz);
} catch (Exception e) {
throw new RuntimeException("Failed to load auto commit settings!", e);
}
//API calls to adminSettingsService are not in transaction, so we can simply evict the cache.
cache.evict(tenantId);
return savedSettings;
}
public boolean delete(TenantId tenantId) {
boolean result = adminSettingsService.deleteAdminSettingsByTenantIdAndKey(tenantId, settingsKey);
cache.evict(tenantId);
return result;
}
}

34
application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsCaffeineCache.java

@ -0,0 +1,34 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.autocommit;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "caffeine", matchIfMissing = true)
@Service("AutoCommitSettingsCache")
public class AutoCommitSettingsCaffeineCache extends CaffeineTbTransactionalCache<TenantId, AutoCommitSettings> {
public AutoCommitSettingsCaffeineCache(CacheManager cacheManager) {
super(cacheManager, CacheConstants.AUTO_COMMIT_SETTINGS_CACHE);
}
}

36
application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.autocommit;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.CacheSpecsMap;
import org.thingsboard.server.cache.RedisTbTransactionalCache;
import org.thingsboard.server.cache.TBRedisCacheConfiguration;
import org.thingsboard.server.cache.TbRedisSerializer;
import org.thingsboard.server.common.data.CacheConstants;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings;
@ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis")
@Service("AutoCommitSettingsCache")
public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache<TenantId, AutoCommitSettings> {
public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) {
super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>());
}
}

36
application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/DefaultTbAutoCommitSettingsService.java

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.autocommit;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.TbAbstractVersionControlSettingsService;
@Service
@TbCoreComponent
public class DefaultTbAutoCommitSettingsService extends TbAbstractVersionControlSettingsService<AutoCommitSettings> implements TbAutoCommitSettingsService {
public static final String SETTINGS_KEY = "autoCommitSettings";
public DefaultTbAutoCommitSettingsService(AdminSettingsService adminSettingsService, TbTransactionalCache<TenantId, AutoCommitSettings> cache) {
super(adminSettingsService, cache, AutoCommitSettings.class, SETTINGS_KEY);
}
}

30
application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/TbAutoCommitSettingsService.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.autocommit;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
public interface TbAutoCommitSettingsService {
AutoCommitSettings get(TenantId tenantId);
AutoCommitSettings save(TenantId tenantId, AutoCommitSettings settings);
boolean delete(TenantId tenantId);
}

30
application/src/main/java/org/thingsboard/server/service/sync/vc/data/ClearRepositoryGitRequest.java

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data;
import org.thingsboard.server.common.data.id.TenantId;
public class ClearRepositoryGitRequest extends VoidGitRequest {
public ClearRepositoryGitRequest(TenantId tenantId) {
super(tenantId);
}
public boolean requiresSettings() {
return false;
}
}

37
application/src/main/java/org/thingsboard/server/service/sync/vc/data/CommitGitRequest.java

@ -0,0 +1,37 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data;
import lombok.Getter;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import java.util.UUID;
public class CommitGitRequest extends PendingGitRequest<VersionCreationResult> {
@Getter
private final UUID txId;
private final VersionCreateRequest request;
public CommitGitRequest(TenantId tenantId, VersionCreateRequest request) {
super(tenantId);
this.txId = UUID.randomUUID();
this.request = request;
}
}

43
application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.sync.vc.data;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.HashMap;
import java.util.Map;
public class ComplexEntitiesExportCtx extends EntitiesExportCtx<ComplexVersionCreateRequest> {
private final Map<EntityType, EntityExportSettings> settings = new HashMap<>();
public ComplexEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, ComplexVersionCreateRequest request) {
super(user, commit, request);
request.getEntityTypes().forEach((type, config) -> settings.put(type, buildExportSettings(config)));
}
public EntityExportSettings getSettings(EntityType entityType) {
return settings.get(entityType);
}
@Override
public EntityExportSettings getSettings() {
throw new RuntimeException("Not implemented. Use EntityTypeExportCtx instead!");
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save